VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65134

Last change on this file since 65134 was 65134, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Comment nits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 584.0 KB
Line 
1/* $Id: HMVMXR0.cpp 65134 2017-01-05 05:46:09Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923 }
924
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928#endif
929}
930
931
932/**
933 * Worker function to allocate VT-x related VM structures.
934 *
935 * @returns IPRT status code.
936 * @param pVM The cross context VM structure.
937 */
938static int hmR0VmxStructsAlloc(PVM pVM)
939{
940 /*
941 * Initialize members up-front so we can cleanup properly on allocation failure.
942 */
943#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955#endif
956 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959 for (VMCPUID i = 0; i < pVM->cCpus; i++)
960 {
961 PVMCPU pVCpu = &pVM->aCpus[i];
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
965 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
967 }
968#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
969#undef VMXLOCAL_INIT_VM_MEMOBJ
970
971 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
972 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
973 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
974 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
975
976 /*
977 * Allocate all the VT-x structures.
978 */
979 int rc = VINF_SUCCESS;
980#ifdef VBOX_WITH_CRASHDUMP_MAGIC
981 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
982 if (RT_FAILURE(rc))
983 goto cleanup;
984 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
985 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
986#endif
987
988 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
990 {
991 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
992 &pVM->hm.s.vmx.HCPhysApicAccess);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 }
996
997 /*
998 * Initialize per-VCPU VT-x structures.
999 */
1000 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1001 {
1002 PVMCPU pVCpu = &pVM->aCpus[i];
1003 AssertPtr(pVCpu);
1004
1005 /* Allocate the VM control structure (VMCS). */
1006 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1007 if (RT_FAILURE(rc))
1008 goto cleanup;
1009
1010 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1011 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 }
1018
1019 /*
1020 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021 * transparent accesses of specific MSRs.
1022 *
1023 * If the condition for enabling MSR bitmaps changes here, don't forget to
1024 * update HMAreMsrBitmapsAvailable().
1025 */
1026 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027 {
1028 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030 if (RT_FAILURE(rc))
1031 goto cleanup;
1032 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033 }
1034
1035 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039
1040 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1041 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044 }
1045
1046 return VINF_SUCCESS;
1047
1048cleanup:
1049 hmR0VmxStructsFree(pVM);
1050 return rc;
1051}
1052
1053
1054/**
1055 * Does global VT-x initialization (called during module initialization).
1056 *
1057 * @returns VBox status code.
1058 */
1059VMMR0DECL(int) VMXR0GlobalInit(void)
1060{
1061#ifdef HMVMX_USE_FUNCTION_TABLE
1062 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063# ifdef VBOX_STRICT
1064 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065 Assert(g_apfnVMExitHandlers[i]);
1066# endif
1067#endif
1068 return VINF_SUCCESS;
1069}
1070
1071
1072/**
1073 * Does global VT-x termination (called during module termination).
1074 */
1075VMMR0DECL(void) VMXR0GlobalTerm()
1076{
1077 /* Nothing to do currently. */
1078}
1079
1080
1081/**
1082 * Sets up and activates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pVM The cross context VM structure. Can be
1087 * NULL after a host resume operation.
1088 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089 * fEnabledByHost is @c true).
1090 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091 * @a fEnabledByHost is @c true).
1092 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093 * enable VT-x on the host.
1094 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1095 */
1096VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097 void *pvMsrs)
1098{
1099 Assert(pCpu);
1100 Assert(pvMsrs);
1101 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103 /* Enable VT-x if it's not already enabled by the host. */
1104 if (!fEnabledByHost)
1105 {
1106 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107 if (RT_FAILURE(rc))
1108 return rc;
1109 }
1110
1111 /*
1112 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119 pCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 NOREF(pCpu);
1145 NOREF(pvCpuPage);
1146 NOREF(HCPhysCpuPage);
1147
1148 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149 return hmR0VmxLeaveRootMode();
1150}
1151
1152
1153/**
1154 * Sets the permission bits for the specified MSR in the MSR bitmap.
1155 *
1156 * @param pVCpu The cross context virtual CPU structure.
1157 * @param uMsr The MSR value.
1158 * @param enmRead Whether reading this MSR causes a VM-exit.
1159 * @param enmWrite Whether writing this MSR causes a VM-exit.
1160 */
1161static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162{
1163 int32_t iBit;
1164 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166 /*
1167 * Layout:
1168 * 0x000 - 0x3ff - Low MSR read bits
1169 * 0x400 - 0x7ff - High MSR read bits
1170 * 0x800 - 0xbff - Low MSR write bits
1171 * 0xc00 - 0xfff - High MSR write bits
1172 */
1173 if (uMsr <= 0x00001FFF)
1174 iBit = uMsr;
1175 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176 {
1177 iBit = uMsr - UINT32_C(0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183 Assert(iBit <= 0x1fff);
1184 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185 ASMBitSet(pbMsrBitmap, iBit);
1186 else
1187 ASMBitClear(pbMsrBitmap, iBit);
1188
1189 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191 else
1192 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193}
1194
1195
1196#ifdef VBOX_STRICT
1197/**
1198 * Gets the permission bits for the specified MSR in the MSR bitmap.
1199 *
1200 * @returns VBox status code.
1201 * @retval VINF_SUCCESS if the specified MSR is found.
1202 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1203 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204 *
1205 * @param pVCpu The cross context virtual CPU structure.
1206 * @param uMsr The MSR.
1207 * @param penmRead Where to store the read permissions.
1208 * @param penmWrite Where to store the write permissions.
1209 */
1210static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211{
1212 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214 int32_t iBit;
1215 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217 /* See hmR0VmxSetMsrPermission() for the layout. */
1218 if (uMsr <= 0x00001FFF)
1219 iBit = uMsr;
1220 else if ( uMsr >= 0xC0000000
1221 && uMsr <= 0xC0001FFF)
1222 {
1223 iBit = (uMsr - 0xC0000000);
1224 pbMsrBitmap += 0x400;
1225 }
1226 else
1227 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229 Assert(iBit <= 0x1fff);
1230 if (ASMBitTest(pbMsrBitmap, iBit))
1231 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1232 else
1233 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237 else
1238 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239 return VINF_SUCCESS;
1240}
1241#endif /* VBOX_STRICT */
1242
1243
1244/**
1245 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246 * area.
1247 *
1248 * @returns VBox status code.
1249 * @param pVCpu The cross context virtual CPU structure.
1250 * @param cMsrs The number of MSRs.
1251 */
1252DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253{
1254 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257 {
1258 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261 }
1262
1263 /* Update number of guest MSRs to load/store across the world-switch. */
1264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269 AssertRCReturn(rc, rc);
1270
1271 /* Update the VCPU's copy of the MSR count. */
1272 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274 return VINF_SUCCESS;
1275}
1276
1277
1278/**
1279 * Adds a new (or updates the value of an existing) guest/host MSR
1280 * pair to be swapped during the world-switch as part of the
1281 * auto-load/store MSR area in the VMCS.
1282 *
1283 * @returns VBox status code.
1284 * @param pVCpu The cross context virtual CPU structure.
1285 * @param uMsr The MSR.
1286 * @param uGuestMsrValue Value of the guest MSR.
1287 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1288 * necessary.
1289 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290 * its value was updated. Optional, can be NULL.
1291 */
1292static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293 bool *pfAddedAndUpdated)
1294{
1295 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297 uint32_t i;
1298 for (i = 0; i < cMsrs; i++)
1299 {
1300 if (pGuestMsr->u32Msr == uMsr)
1301 break;
1302 pGuestMsr++;
1303 }
1304
1305 bool fAdded = false;
1306 if (i == cMsrs)
1307 {
1308 ++cMsrs;
1309 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316 fAdded = true;
1317 }
1318
1319 /* Update the MSR values in the auto-load/store MSR area. */
1320 pGuestMsr->u32Msr = uMsr;
1321 pGuestMsr->u64Value = uGuestMsrValue;
1322
1323 /* Create/update the MSR slot in the host MSR area. */
1324 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325 pHostMsr += i;
1326 pHostMsr->u32Msr = uMsr;
1327
1328 /*
1329 * Update the host MSR only when requested by the caller AND when we're
1330 * adding it to the auto-load/store area. Otherwise, it would have been
1331 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332 */
1333 bool fUpdatedMsrValue = false;
1334 if ( fAdded
1335 && fUpdateHostMsr)
1336 {
1337 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340 fUpdatedMsrValue = true;
1341 }
1342
1343 if (pfAddedAndUpdated)
1344 *pfAddedAndUpdated = fUpdatedMsrValue;
1345 return VINF_SUCCESS;
1346}
1347
1348
1349/**
1350 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1351 * auto-load/store MSR area in the VMCS.
1352 *
1353 * @returns VBox status code.
1354 * @param pVCpu The cross context virtual CPU structure.
1355 * @param uMsr The MSR.
1356 */
1357static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358{
1359 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361 for (uint32_t i = 0; i < cMsrs; i++)
1362 {
1363 /* Find the MSR. */
1364 if (pGuestMsr->u32Msr == uMsr)
1365 {
1366 /* If it's the last MSR, simply reduce the count. */
1367 if (i == cMsrs - 1)
1368 {
1369 --cMsrs;
1370 break;
1371 }
1372
1373 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375 pLastGuestMsr += cMsrs - 1;
1376 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381 pLastHostMsr += cMsrs - 1;
1382 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383 pHostMsr->u64Value = pLastHostMsr->u64Value;
1384 --cMsrs;
1385 break;
1386 }
1387 pGuestMsr++;
1388 }
1389
1390 /* Update the VMCS if the count changed (meaning the MSR was found). */
1391 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392 {
1393 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394 AssertRCReturn(rc, rc);
1395
1396 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401 return VINF_SUCCESS;
1402 }
1403
1404 return VERR_NOT_FOUND;
1405}
1406
1407
1408/**
1409 * Checks if the specified guest MSR is part of the auto-load/store area in
1410 * the VMCS.
1411 *
1412 * @returns true if found, false otherwise.
1413 * @param pVCpu The cross context virtual CPU structure.
1414 * @param uMsr The MSR to find.
1415 */
1416static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417{
1418 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422 {
1423 if (pGuestMsr->u32Msr == uMsr)
1424 return true;
1425 }
1426 return false;
1427}
1428
1429
1430/**
1431 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432 *
1433 * @param pVCpu The cross context virtual CPU structure.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445 {
1446 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448 /*
1449 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451 */
1452 if (pHostMsr->u32Msr == MSR_K6_EFER)
1453 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454 else
1455 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456 }
1457
1458 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459}
1460
1461
1462/**
1463 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1464 * perform lazy restoration of the host MSRs while leaving VT-x.
1465 *
1466 * @param pVCpu The cross context virtual CPU structure.
1467 *
1468 * @remarks No-long-jump zone!!!
1469 */
1470static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471{
1472 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474 /*
1475 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476 */
1477 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478 {
1479 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480#if HC_ARCH_BITS == 64
1481 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482 {
1483 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487 }
1488#endif
1489 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1490 }
1491}
1492
1493
1494/**
1495 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1496 * lazily while leaving VT-x.
1497 *
1498 * @returns true if it does, false otherwise.
1499 * @param pVCpu The cross context virtual CPU structure.
1500 * @param uMsr The MSR to check.
1501 */
1502static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503{
1504 NOREF(pVCpu);
1505#if HC_ARCH_BITS == 64
1506 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507 {
1508 switch (uMsr)
1509 {
1510 case MSR_K8_LSTAR:
1511 case MSR_K6_STAR:
1512 case MSR_K8_SF_MASK:
1513 case MSR_K8_KERNEL_GS_BASE:
1514 return true;
1515 }
1516 }
1517#else
1518 RT_NOREF(pVCpu, uMsr);
1519#endif
1520 return false;
1521}
1522
1523
1524/**
1525 * Saves a set of guest MSRs back into the guest-CPU context.
1526 *
1527 * @param pVCpu The cross context virtual CPU structure.
1528 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529 * out-of-sync. Make sure to update the required fields
1530 * before using them.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540 {
1541 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542#if HC_ARCH_BITS == 64
1543 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544 {
1545 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549 }
1550#else
1551 NOREF(pMixedCtx);
1552#endif
1553 }
1554}
1555
1556
1557/**
1558 * Loads a set of guests MSRs to allow read/passthru to the guest.
1559 *
1560 * The name of this function is slightly confusing. This function does NOT
1561 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562 * common prefix for functions dealing with "lazy restoration" of the shared
1563 * MSRs.
1564 *
1565 * @param pVCpu The cross context virtual CPU structure.
1566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567 * out-of-sync. Make sure to update the required fields
1568 * before using them.
1569 *
1570 * @remarks No-long-jump zone!!!
1571 */
1572static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573{
1574 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1578 do { \
1579 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1580 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1581 else \
1582 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1583 } while (0)
1584
1585 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1586 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1587 {
1588#if HC_ARCH_BITS == 64
1589 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1590 {
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1593 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1594 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1595 }
1596#else
1597 RT_NOREF(pMixedCtx);
1598#endif
1599 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1600 }
1601
1602#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1603}
1604
1605
1606/**
1607 * Performs lazy restoration of the set of host MSRs if they were previously
1608 * loaded with guest MSR values.
1609 *
1610 * @param pVCpu The cross context virtual CPU structure.
1611 *
1612 * @remarks No-long-jump zone!!!
1613 * @remarks The guest MSRs should have been saved back into the guest-CPU
1614 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1615 */
1616static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1617{
1618 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1620
1621 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1622 {
1623 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1624#if HC_ARCH_BITS == 64
1625 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1626 {
1627 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1628 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1629 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1630 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1631 }
1632#endif
1633 }
1634 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1635}
1636
1637
1638/**
1639 * Verifies that our cached values of the VMCS controls are all
1640 * consistent with what's actually present in the VMCS.
1641 *
1642 * @returns VBox status code.
1643 * @param pVCpu The cross context virtual CPU structure.
1644 */
1645static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1646{
1647 uint32_t u32Val;
1648 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1651 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1652
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1656 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1657
1658 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1659 AssertRCReturn(rc, rc);
1660 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1661 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1662
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1664 AssertRCReturn(rc, rc);
1665 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1666 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1667
1668 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1669 {
1670 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1671 AssertRCReturn(rc, rc);
1672 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1673 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1674 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1675 }
1676
1677 return VINF_SUCCESS;
1678}
1679
1680
1681#ifdef VBOX_STRICT
1682/**
1683 * Verifies that our cached host EFER value has not changed
1684 * since we cached it.
1685 *
1686 * @param pVCpu The cross context virtual CPU structure.
1687 */
1688static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1689{
1690 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1691
1692 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1693 {
1694 uint64_t u64Val;
1695 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1696 AssertRC(rc);
1697
1698 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1699 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1700 }
1701}
1702
1703
1704/**
1705 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1706 * VMCS are correct.
1707 *
1708 * @param pVCpu The cross context virtual CPU structure.
1709 */
1710static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1711{
1712 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1713
1714 /* Verify MSR counts in the VMCS are what we think it should be. */
1715 uint32_t cMsrs;
1716 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1717 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1718
1719 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1720 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1723 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1726 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1727 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1728 {
1729 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1730 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1731 pGuestMsr->u32Msr, cMsrs));
1732
1733 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1734 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1735 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1736
1737 /* Verify that the permissions are as expected in the MSR bitmap. */
1738 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1739 {
1740 VMXMSREXITREAD enmRead;
1741 VMXMSREXITWRITE enmWrite;
1742 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1743 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1744 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1745 {
1746 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1747 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1748 }
1749 else
1750 {
1751 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 }
1756 }
1757 }
1758}
1759#endif /* VBOX_STRICT */
1760
1761
1762/**
1763 * Flushes the TLB using EPT.
1764 *
1765 * @returns VBox status code.
1766 * @param pVCpu The cross context virtual CPU structure of the calling
1767 * EMT. Can be NULL depending on @a enmFlush.
1768 * @param enmFlush Type of flush.
1769 *
1770 * @remarks Caller is responsible for making sure this function is called only
1771 * when NestedPaging is supported and providing @a enmFlush that is
1772 * supported by the CPU.
1773 * @remarks Can be called with interrupts disabled.
1774 */
1775static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1776{
1777 uint64_t au64Descriptor[2];
1778 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1779 au64Descriptor[0] = 0;
1780 else
1781 {
1782 Assert(pVCpu);
1783 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1784 }
1785 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1786
1787 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1788 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1789 rc));
1790 if ( RT_SUCCESS(rc)
1791 && pVCpu)
1792 {
1793 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1794 }
1795}
1796
1797
1798/**
1799 * Flushes the TLB using VPID.
1800 *
1801 * @returns VBox status code.
1802 * @param pVM The cross context VM structure.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmFlush.
1805 * @param enmFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1812{
1813 NOREF(pVM);
1814 AssertPtr(pVM);
1815 Assert(pVM->hm.s.vmx.fVpid);
1816
1817 uint64_t au64Descriptor[2];
1818 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1819 {
1820 au64Descriptor[0] = 0;
1821 au64Descriptor[1] = 0;
1822 }
1823 else
1824 {
1825 AssertPtr(pVCpu);
1826 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1828 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1829 au64Descriptor[1] = GCPtr;
1830 }
1831
1832 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1833 AssertMsg(rc == VINF_SUCCESS,
1834 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1835 if ( RT_SUCCESS(rc)
1836 && pVCpu)
1837 {
1838 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1839 }
1840}
1841
1842
1843/**
1844 * Invalidates a guest page by guest virtual address. Only relevant for
1845 * EPT/VPID, otherwise there is nothing really to invalidate.
1846 *
1847 * @returns VBox status code.
1848 * @param pVM The cross context VM structure.
1849 * @param pVCpu The cross context virtual CPU structure.
1850 * @param GCVirt Guest virtual address of the page to invalidate.
1851 */
1852VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1853{
1854 AssertPtr(pVM);
1855 AssertPtr(pVCpu);
1856 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1857
1858 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1859 if (!fFlushPending)
1860 {
1861 /*
1862 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1863 * See @bugref{6043} and @bugref{6177}.
1864 *
1865 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1866 * function maybe called in a loop with individual addresses.
1867 */
1868 if (pVM->hm.s.vmx.fVpid)
1869 {
1870 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1871 {
1872 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1873 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1874 }
1875 else
1876 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1877 }
1878 else if (pVM->hm.s.fNestedPaging)
1879 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1880 }
1881
1882 return VINF_SUCCESS;
1883}
1884
1885
1886/**
1887 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1888 * otherwise there is nothing really to invalidate.
1889 *
1890 * @returns VBox status code.
1891 * @param pVM The cross context VM structure.
1892 * @param pVCpu The cross context virtual CPU structure.
1893 * @param GCPhys Guest physical address of the page to invalidate.
1894 */
1895VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1896{
1897 NOREF(pVM); NOREF(GCPhys);
1898 LogFlowFunc(("%RGp\n", GCPhys));
1899
1900 /*
1901 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1902 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1903 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1904 */
1905 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1906 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1907 return VINF_SUCCESS;
1908}
1909
1910
1911/**
1912 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1913 * case where neither EPT nor VPID is supported by the CPU.
1914 *
1915 * @param pVM The cross context VM structure.
1916 * @param pVCpu The cross context virtual CPU structure.
1917 * @param pCpu Pointer to the global HM struct.
1918 *
1919 * @remarks Called with interrupts disabled.
1920 */
1921static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922{
1923 AssertPtr(pVCpu);
1924 AssertPtr(pCpu);
1925 NOREF(pVM);
1926
1927 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1928
1929 Assert(pCpu->idCpu != NIL_RTCPUID);
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932 pVCpu->hm.s.fForceTLBFlush = false;
1933 return;
1934}
1935
1936
1937/**
1938 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1939 *
1940 * @param pVM The cross context VM structure.
1941 * @param pVCpu The cross context virtual CPU structure.
1942 * @param pCpu Pointer to the global HM CPU struct.
1943 * @remarks All references to "ASID" in this function pertains to "VPID" in
1944 * Intel's nomenclature. The reason is, to avoid confusion in compare
1945 * statements since the host-CPU copies are named "ASID".
1946 *
1947 * @remarks Called with interrupts disabled.
1948 */
1949static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1950{
1951#ifdef VBOX_WITH_STATISTICS
1952 bool fTlbFlushed = false;
1953# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1954# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1955 if (!fTlbFlushed) \
1956 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1957 } while (0)
1958#else
1959# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1960# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1961#endif
1962
1963 AssertPtr(pVM);
1964 AssertPtr(pCpu);
1965 AssertPtr(pVCpu);
1966 Assert(pCpu->idCpu != NIL_RTCPUID);
1967
1968 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1969 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1970 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1971
1972 /*
1973 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1974 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1975 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1976 */
1977 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1978 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1979 {
1980 ++pCpu->uCurrentAsid;
1981 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1982 {
1983 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1984 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1985 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1986 }
1987
1988 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1989 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1990 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1991
1992 /*
1993 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1994 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1995 */
1996 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1997 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1998 HMVMX_SET_TAGGED_TLB_FLUSHED();
1999 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2000 }
2001
2002 /* Check for explicit TLB flushes. */
2003 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2004 {
2005 /*
2006 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2007 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2008 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2009 * but not guest-physical mappings.
2010 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2011 */
2012 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2014 HMVMX_SET_TAGGED_TLB_FLUSHED();
2015 }
2016
2017 pVCpu->hm.s.fForceTLBFlush = false;
2018 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2019
2020 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2021 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2022 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2023 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2024 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2025 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2026 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2027 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2028 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2029
2030 /* Update VMCS with the VPID. */
2031 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2032 AssertRC(rc);
2033
2034#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2035}
2036
2037
2038/**
2039 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2040 *
2041 * @returns VBox status code.
2042 * @param pVM The cross context VM structure.
2043 * @param pVCpu The cross context virtual CPU structure.
2044 * @param pCpu Pointer to the global HM CPU struct.
2045 *
2046 * @remarks Called with interrupts disabled.
2047 */
2048static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2049{
2050 AssertPtr(pVM);
2051 AssertPtr(pVCpu);
2052 AssertPtr(pCpu);
2053 Assert(pCpu->idCpu != NIL_RTCPUID);
2054 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2055 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2056
2057 /*
2058 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2059 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2060 */
2061 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2062 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2063 {
2064 pVCpu->hm.s.fForceTLBFlush = true;
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2066 }
2067
2068 /* Check for explicit TLB flushes. */
2069 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2070 {
2071 pVCpu->hm.s.fForceTLBFlush = true;
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2073 }
2074
2075 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2076 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2077
2078 if (pVCpu->hm.s.fForceTLBFlush)
2079 {
2080 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2081 pVCpu->hm.s.fForceTLBFlush = false;
2082 }
2083}
2084
2085
2086/**
2087 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2088 *
2089 * @returns VBox status code.
2090 * @param pVM The cross context VM structure.
2091 * @param pVCpu The cross context virtual CPU structure.
2092 * @param pCpu Pointer to the global HM CPU struct.
2093 *
2094 * @remarks Called with interrupts disabled.
2095 */
2096static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097{
2098 AssertPtr(pVM);
2099 AssertPtr(pVCpu);
2100 AssertPtr(pCpu);
2101 Assert(pCpu->idCpu != NIL_RTCPUID);
2102 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2103 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2104
2105 /*
2106 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2107 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2108 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2109 */
2110 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2111 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2112 {
2113 pVCpu->hm.s.fForceTLBFlush = true;
2114 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2115 }
2116
2117 /* Check for explicit TLB flushes. */
2118 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2119 {
2120 /*
2121 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2122 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2123 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2124 */
2125 pVCpu->hm.s.fForceTLBFlush = true;
2126 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2127 }
2128
2129 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2130 if (pVCpu->hm.s.fForceTLBFlush)
2131 {
2132 ++pCpu->uCurrentAsid;
2133 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2134 {
2135 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2136 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2137 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2138 }
2139
2140 pVCpu->hm.s.fForceTLBFlush = false;
2141 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2142 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2143 if (pCpu->fFlushAsidBeforeUse)
2144 {
2145 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2146 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2147 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2148 {
2149 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2150 pCpu->fFlushAsidBeforeUse = false;
2151 }
2152 else
2153 {
2154 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2155 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2156 }
2157 }
2158 }
2159
2160 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2161 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2162 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2163 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2164 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2165 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2166 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2167
2168 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2169 AssertRC(rc);
2170}
2171
2172
2173/**
2174 * Flushes the guest TLB entry based on CPU capabilities.
2175 *
2176 * @param pVCpu The cross context virtual CPU structure.
2177 * @param pCpu Pointer to the global HM CPU struct.
2178 */
2179DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2180{
2181#ifdef HMVMX_ALWAYS_FLUSH_TLB
2182 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2183#endif
2184 PVM pVM = pVCpu->CTX_SUFF(pVM);
2185 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2186 {
2187 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2189 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2190 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2191 default:
2192 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2193 break;
2194 }
2195
2196 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2197}
2198
2199
2200/**
2201 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2202 * TLB entries from the host TLB before VM-entry.
2203 *
2204 * @returns VBox status code.
2205 * @param pVM The cross context VM structure.
2206 */
2207static int hmR0VmxSetupTaggedTlb(PVM pVM)
2208{
2209 /*
2210 * Determine optimal flush type for Nested Paging.
2211 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2212 * guest execution (see hmR3InitFinalizeR0()).
2213 */
2214 if (pVM->hm.s.fNestedPaging)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2217 {
2218 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2220 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2221 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* Make sure the write-back cacheable memory type for EPT is supported. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237
2238 /* EPT requires a page-walk length of 4. */
2239 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2240 {
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2252 }
2253 }
2254
2255 /*
2256 * Determine optimal flush type for VPID.
2257 */
2258 if (pVM->hm.s.vmx.fVpid)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2261 {
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2264 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2266 else
2267 {
2268 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2272 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277 else
2278 {
2279 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2280 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282 pVM->hm.s.vmx.fVpid = false;
2283 }
2284 }
2285
2286 /*
2287 * Setup the handler for flushing tagged-TLBs.
2288 */
2289 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2291 else if (pVM->hm.s.fNestedPaging)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2293 else if (pVM->hm.s.vmx.fVpid)
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2295 else
2296 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2297 return VINF_SUCCESS;
2298}
2299
2300
2301/**
2302 * Sets up pin-based VM-execution controls in the VMCS.
2303 *
2304 * @returns VBox status code.
2305 * @param pVM The cross context VM structure.
2306 * @param pVCpu The cross context virtual CPU structure.
2307 */
2308static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2309{
2310 AssertPtr(pVM);
2311 AssertPtr(pVCpu);
2312
2313 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2314 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2315
2316 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2317 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2318
2319 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2320 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2321
2322 /* Enable the VMX preemption timer. */
2323 if (pVM->hm.s.vmx.fUsePreemptTimer)
2324 {
2325 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2326 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2327 }
2328
2329#if 0
2330 /* Enable posted-interrupt processing. */
2331 if (pVM->hm.s.fPostedIntrs)
2332 {
2333 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2334 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2336 }
2337#endif
2338
2339 if ((val & zap) != val)
2340 {
2341 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2342 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2343 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2344 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2345 }
2346
2347 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2348 AssertRCReturn(rc, rc);
2349
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM The cross context VM structure.
2360 * @param pVCpu The cross context virtual CPU structure.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2457 }
2458
2459 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2460 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2462
2463 if ((val & zap) != val)
2464 {
2465 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2466 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2467 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2468 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2469 }
2470
2471 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2472 AssertRCReturn(rc, rc);
2473
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506#if 0
2507 if (pVM->hm.s.fVirtApicRegs)
2508 {
2509 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2511
2512 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2514 }
2515#endif
2516
2517 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2518 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2519 * done dynamically. */
2520 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2521 {
2522 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2523 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2525 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2526 AssertRCReturn(rc, rc);
2527 }
2528
2529 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2530 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2531
2532 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2533 && pVM->hm.s.vmx.cPleGapTicks
2534 && pVM->hm.s.vmx.cPleWindowTicks)
2535 {
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2537
2538 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2539 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2540 AssertRCReturn(rc, rc);
2541 }
2542
2543 if ((val & zap) != val)
2544 {
2545 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2546 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2547 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2548 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2549 }
2550
2551 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2552 AssertRCReturn(rc, rc);
2553
2554 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2555 }
2556 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2557 {
2558 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2559 "available\n"));
2560 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2561 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562 }
2563
2564 return VINF_SUCCESS;
2565}
2566
2567
2568/**
2569 * Sets up miscellaneous (everything other than Pin & Processor-based
2570 * VM-execution) control fields in the VMCS.
2571 *
2572 * @returns VBox status code.
2573 * @param pVM The cross context VM structure.
2574 * @param pVCpu The cross context virtual CPU structure.
2575 */
2576static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2577{
2578 NOREF(pVM);
2579 AssertPtr(pVM);
2580 AssertPtr(pVCpu);
2581
2582 int rc = VERR_GENERAL_FAILURE;
2583
2584 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2585#if 0
2586 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2588 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2589
2590 /*
2591 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2592 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2593 * We thus use the exception bitmap to control it rather than use both.
2594 */
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2597
2598 /** @todo Explore possibility of using IO-bitmaps. */
2599 /* All IO & IOIO instructions cause VM-exits. */
2600 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2601 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2602
2603 /* Initialize the MSR-bitmap area. */
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2607 AssertRCReturn(rc, rc);
2608#endif
2609
2610 /* Setup MSR auto-load/store area. */
2611 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2615 AssertRCReturn(rc, rc);
2616
2617 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 AssertRCReturn(rc, rc);
2621
2622 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2623 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2624 AssertRCReturn(rc, rc);
2625
2626 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2627#if 0
2628 /* Setup debug controls */
2629 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2630 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2631 AssertRCReturn(rc, rc);
2632#endif
2633
2634 return rc;
2635}
2636
2637
2638/**
2639 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2640 *
2641 * We shall setup those exception intercepts that don't change during the
2642 * lifetime of the VM here. The rest are done dynamically while loading the
2643 * guest state.
2644 *
2645 * @returns VBox status code.
2646 * @param pVM The cross context VM structure.
2647 * @param pVCpu The cross context virtual CPU structure.
2648 */
2649static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2650{
2651 AssertPtr(pVM);
2652 AssertPtr(pVCpu);
2653
2654 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2655
2656 uint32_t u32XcptBitmap = 0;
2657
2658 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2659 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2660
2661 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2662 and writes, and because recursive #DBs can cause the CPU hang, we must always
2663 intercept #DB. */
2664 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2665
2666 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2667 if (!pVM->hm.s.fNestedPaging)
2668 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2669
2670 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2671 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2672 AssertRCReturn(rc, rc);
2673 return rc;
2674}
2675
2676
2677/**
2678 * Sets up the initial guest-state mask. The guest-state mask is consulted
2679 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2680 * for the nested virtualization case (as it would cause a VM-exit).
2681 *
2682 * @param pVCpu The cross context virtual CPU structure.
2683 */
2684static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2685{
2686 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2687 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2688 return VINF_SUCCESS;
2689}
2690
2691
2692/**
2693 * Does per-VM VT-x initialization.
2694 *
2695 * @returns VBox status code.
2696 * @param pVM The cross context VM structure.
2697 */
2698VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2699{
2700 LogFlowFunc(("pVM=%p\n", pVM));
2701
2702 int rc = hmR0VmxStructsAlloc(pVM);
2703 if (RT_FAILURE(rc))
2704 {
2705 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2706 return rc;
2707 }
2708
2709 return VINF_SUCCESS;
2710}
2711
2712
2713/**
2714 * Does per-VM VT-x termination.
2715 *
2716 * @returns VBox status code.
2717 * @param pVM The cross context VM structure.
2718 */
2719VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2720{
2721 LogFlowFunc(("pVM=%p\n", pVM));
2722
2723#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2724 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2725 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2726#endif
2727 hmR0VmxStructsFree(pVM);
2728 return VINF_SUCCESS;
2729}
2730
2731
2732/**
2733 * Sets up the VM for execution under VT-x.
2734 * This function is only called once per-VM during initialization.
2735 *
2736 * @returns VBox status code.
2737 * @param pVM The cross context VM structure.
2738 */
2739VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2740{
2741 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2742 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2743
2744 LogFlowFunc(("pVM=%p\n", pVM));
2745
2746 /*
2747 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2748 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2749 */
2750 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2751 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2752 || !pVM->hm.s.vmx.pRealModeTSS))
2753 {
2754 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2755 return VERR_INTERNAL_ERROR;
2756 }
2757
2758 /* Initialize these always, see hmR3InitFinalizeR0().*/
2759 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2760 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2761
2762 /* Setup the tagged-TLB flush handlers. */
2763 int rc = hmR0VmxSetupTaggedTlb(pVM);
2764 if (RT_FAILURE(rc))
2765 {
2766 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2767 return rc;
2768 }
2769
2770 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2771 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2772#if HC_ARCH_BITS == 64
2773 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2774 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2775 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2776 {
2777 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2778 }
2779#endif
2780
2781 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2782 RTCCUINTREG uHostCR4 = ASMGetCR4();
2783 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2784 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2785
2786 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2787 {
2788 PVMCPU pVCpu = &pVM->aCpus[i];
2789 AssertPtr(pVCpu);
2790 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2791
2792 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2793 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2794
2795 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2796 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2797 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2798
2799 /* Set revision dword at the beginning of the VMCS structure. */
2800 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2801
2802 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2803 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 /* Load this VMCS as the current VMCS. */
2808 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2826 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2827
2828 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2829 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2830 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2831
2832#if HC_ARCH_BITS == 32
2833 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2834 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2835 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2836#endif
2837
2838 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2839 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2840 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2841 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2842
2843 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2844
2845 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2846 }
2847
2848 return VINF_SUCCESS;
2849}
2850
2851
2852/**
2853 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2854 * the VMCS.
2855 *
2856 * @returns VBox status code.
2857 * @param pVM The cross context VM structure.
2858 * @param pVCpu The cross context virtual CPU structure.
2859 */
2860DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2861{
2862 NOREF(pVM); NOREF(pVCpu);
2863
2864 RTCCUINTREG uReg = ASMGetCR0();
2865 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2866 AssertRCReturn(rc, rc);
2867
2868 uReg = ASMGetCR3();
2869 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2870 AssertRCReturn(rc, rc);
2871
2872 uReg = ASMGetCR4();
2873 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2874 AssertRCReturn(rc, rc);
2875 return rc;
2876}
2877
2878
2879#if HC_ARCH_BITS == 64
2880/**
2881 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2882 * requirements. See hmR0VmxSaveHostSegmentRegs().
2883 */
2884# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2885 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2886 { \
2887 bool fValidSelector = true; \
2888 if ((selValue) & X86_SEL_LDT) \
2889 { \
2890 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2891 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2892 } \
2893 if (fValidSelector) \
2894 { \
2895 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2896 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2897 } \
2898 (selValue) = 0; \
2899 }
2900#endif
2901
2902
2903/**
2904 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2905 * the host-state area in the VMCS.
2906 *
2907 * @returns VBox status code.
2908 * @param pVM The cross context VM structure.
2909 * @param pVCpu The cross context virtual CPU structure.
2910 */
2911DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2912{
2913 int rc = VERR_INTERNAL_ERROR_5;
2914
2915#if HC_ARCH_BITS == 64
2916 /*
2917 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2918 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2919 *
2920 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2921 * Was observed booting Solaris10u10 32-bit guest.
2922 */
2923 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2924 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2925 {
2926 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2927 pVCpu->idCpu));
2928 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2929 }
2930 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2931#else
2932 RT_NOREF(pVCpu);
2933#endif
2934
2935 /*
2936 * Host DS, ES, FS and GS segment registers.
2937 */
2938#if HC_ARCH_BITS == 64
2939 RTSEL uSelDS = ASMGetDS();
2940 RTSEL uSelES = ASMGetES();
2941 RTSEL uSelFS = ASMGetFS();
2942 RTSEL uSelGS = ASMGetGS();
2943#else
2944 RTSEL uSelDS = 0;
2945 RTSEL uSelES = 0;
2946 RTSEL uSelFS = 0;
2947 RTSEL uSelGS = 0;
2948#endif
2949
2950 /*
2951 * Host CS and SS segment registers.
2952 */
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955
2956 /*
2957 * Host TR segment register.
2958 */
2959 RTSEL uSelTR = ASMGetTR();
2960
2961#if HC_ARCH_BITS == 64
2962 /*
2963 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2964 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2965 */
2966 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2967 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2968 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2969 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2970# undef VMXLOCAL_ADJUST_HOST_SEG
2971#endif
2972
2973 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2974 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2975 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2976 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2977 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2978 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2979 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2980 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2981 Assert(uSelCS);
2982 Assert(uSelTR);
2983
2984 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2985#if 0
2986 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2987 Assert(uSelSS != 0);
2988#endif
2989
2990 /* Write these host selector fields into the host-state area in the VMCS. */
2991 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2993#if HC_ARCH_BITS == 64
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2996 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2997 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2998#else
2999 NOREF(uSelDS);
3000 NOREF(uSelES);
3001 NOREF(uSelFS);
3002 NOREF(uSelGS);
3003#endif
3004 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3005 AssertRCReturn(rc, rc);
3006
3007 /*
3008 * Host GDTR and IDTR.
3009 */
3010 RTGDTR Gdtr;
3011 RTIDTR Idtr;
3012 RT_ZERO(Gdtr);
3013 RT_ZERO(Idtr);
3014 ASMGetGDTR(&Gdtr);
3015 ASMGetIDTR(&Idtr);
3016 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3017 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3018 AssertRCReturn(rc, rc);
3019
3020#if HC_ARCH_BITS == 64
3021 /*
3022 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3023 * maximum limit (0xffff) on every VM-exit.
3024 */
3025 if (Gdtr.cbGdt != 0xffff)
3026 {
3027 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3028 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3029 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3030 }
3031
3032 /*
3033 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3034 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3035 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3036 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3037 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3038 * hosts where we are pretty sure it won't cause trouble.
3039 */
3040# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3041 if (Idtr.cbIdt < 0x0fff)
3042# else
3043 if (Idtr.cbIdt != 0xffff)
3044# endif
3045 {
3046 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3047 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3048 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3049 }
3050#endif
3051
3052 /*
3053 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3054 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3055 */
3056 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3057 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3058 VERR_VMX_INVALID_HOST_STATE);
3059
3060 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3061#if HC_ARCH_BITS == 64
3062 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3063
3064 /*
3065 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3066 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3067 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3068 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3069 *
3070 * [1] See Intel spec. 3.5 "System Descriptor Types".
3071 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3072 */
3073 Assert(pDesc->System.u4Type == 11);
3074 if ( pDesc->System.u16LimitLow != 0x67
3075 || pDesc->System.u4LimitHigh)
3076 {
3077 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3078 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3079 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3080 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3081 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3082
3083 /* Store the GDTR here as we need it while restoring TR. */
3084 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3085 }
3086#else
3087 NOREF(pVM);
3088 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3089#endif
3090 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /*
3094 * Host FS base and GS base.
3095 */
3096#if HC_ARCH_BITS == 64
3097 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3098 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3099 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3100 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3101 AssertRCReturn(rc, rc);
3102
3103 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3106 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3107 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3108#endif
3109 return rc;
3110}
3111
3112
3113/**
3114 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3115 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3116 * the host after every successful VM-exit.
3117 *
3118 * @returns VBox status code.
3119 * @param pVM The cross context VM structure.
3120 * @param pVCpu The cross context virtual CPU structure.
3121 *
3122 * @remarks No-long-jump zone!!!
3123 */
3124DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3125{
3126 NOREF(pVM);
3127
3128 AssertPtr(pVCpu);
3129 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3130
3131 /*
3132 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3133 * rather than swapping them on every VM-entry.
3134 */
3135 hmR0VmxLazySaveHostMsrs(pVCpu);
3136
3137 /*
3138 * Host Sysenter MSRs.
3139 */
3140 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3141#if HC_ARCH_BITS == 32
3142 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3143 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3144#else
3145 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3146 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3147#endif
3148 AssertRCReturn(rc, rc);
3149
3150 /*
3151 * Host EFER MSR.
3152 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3153 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3154 */
3155 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3156 {
3157 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3158 AssertRCReturn(rc, rc);
3159 }
3160
3161 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3162 * hmR0VmxLoadGuestExitCtls() !! */
3163
3164 return rc;
3165}
3166
3167
3168/**
3169 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3170 *
3171 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3172 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3173 * hmR0VMxLoadGuestEntryCtls().
3174 *
3175 * @returns true if we need to load guest EFER, false otherwise.
3176 * @param pVCpu The cross context virtual CPU structure.
3177 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3178 * out-of-sync. Make sure to update the required fields
3179 * before using them.
3180 *
3181 * @remarks Requires EFER, CR4.
3182 * @remarks No-long-jump zone!!!
3183 */
3184static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3185{
3186#ifdef HMVMX_ALWAYS_SWAP_EFER
3187 return true;
3188#endif
3189
3190#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3191 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3192 if (CPUMIsGuestInLongMode(pVCpu))
3193 return false;
3194#endif
3195
3196 PVM pVM = pVCpu->CTX_SUFF(pVM);
3197 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3198 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3199
3200 /*
3201 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3202 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3203 */
3204 if ( CPUMIsGuestInLongMode(pVCpu)
3205 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3206 {
3207 return true;
3208 }
3209
3210 /*
3211 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3212 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3213 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3214 */
3215 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3216 && (pMixedCtx->cr0 & X86_CR0_PG)
3217 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3218 {
3219 /* Assert that host is PAE capable. */
3220 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3221 return true;
3222 }
3223
3224 /** @todo Check the latest Intel spec. for any other bits,
3225 * like SMEP/SMAP? */
3226 return false;
3227}
3228
3229
3230/**
3231 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3232 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3233 * controls".
3234 *
3235 * @returns VBox status code.
3236 * @param pVCpu The cross context virtual CPU structure.
3237 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3238 * out-of-sync. Make sure to update the required fields
3239 * before using them.
3240 *
3241 * @remarks Requires EFER.
3242 * @remarks No-long-jump zone!!!
3243 */
3244DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3245{
3246 int rc = VINF_SUCCESS;
3247 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3248 {
3249 PVM pVM = pVCpu->CTX_SUFF(pVM);
3250 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3251 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3252
3253 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3254 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3255
3256 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3257 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3261 }
3262 else
3263 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3264
3265 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3266 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3267 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3268 {
3269 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3270 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3271 }
3272
3273 /*
3274 * The following should -not- be set (since we're not in SMM mode):
3275 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3276 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3277 */
3278
3279 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3280 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3281
3282 if ((val & zap) != val)
3283 {
3284 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3285 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3286 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3287 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3288 }
3289
3290 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3291 AssertRCReturn(rc, rc);
3292
3293 pVCpu->hm.s.vmx.u32EntryCtls = val;
3294 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3295 }
3296 return rc;
3297}
3298
3299
3300/**
3301 * Sets up the VM-exit controls in the VMCS.
3302 *
3303 * @returns VBox status code.
3304 * @param pVCpu The cross context virtual CPU structure.
3305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3306 * out-of-sync. Make sure to update the required fields
3307 * before using them.
3308 *
3309 * @remarks Requires EFER.
3310 */
3311DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3312{
3313 NOREF(pMixedCtx);
3314
3315 int rc = VINF_SUCCESS;
3316 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3317 {
3318 PVM pVM = pVCpu->CTX_SUFF(pVM);
3319 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3320 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3321
3322 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3323 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3324
3325 /*
3326 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3327 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3328 */
3329#if HC_ARCH_BITS == 64
3330 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3331 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3332#else
3333 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3334 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3335 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3336 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3337 {
3338 /* The switcher returns to long mode, EFER is managed by the switcher. */
3339 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3340 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3341 }
3342 else
3343 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3344#endif
3345
3346 /* If the newer VMCS fields for managing EFER exists, use it. */
3347 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3348 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3349 {
3350 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3351 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3352 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3353 }
3354
3355 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3356 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3357
3358 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3359 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3360 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3361
3362 if ( pVM->hm.s.vmx.fUsePreemptTimer
3363 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3364 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3365
3366 if ((val & zap) != val)
3367 {
3368 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3369 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3370 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3371 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3372 }
3373
3374 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3375 AssertRCReturn(rc, rc);
3376
3377 pVCpu->hm.s.vmx.u32ExitCtls = val;
3378 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3379 }
3380 return rc;
3381}
3382
3383
3384/**
3385 * Sets the TPR threshold in the VMCS.
3386 *
3387 * @returns VBox status code.
3388 * @param pVCpu The cross context virtual CPU structure.
3389 * @param u32TprThreshold The TPR threshold (task-priority class only).
3390 */
3391DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3392{
3393 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3394 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3395 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3396}
3397
3398
3399/**
3400 * Loads the guest APIC and related state.
3401 *
3402 * @returns VBox status code.
3403 * @param pVCpu The cross context virtual CPU structure.
3404 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3405 * out-of-sync. Make sure to update the required fields
3406 * before using them.
3407 */
3408DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3409{
3410 NOREF(pMixedCtx);
3411
3412 int rc = VINF_SUCCESS;
3413 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3414 {
3415 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3416 && APICIsEnabled(pVCpu))
3417 {
3418 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3419 {
3420 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3421
3422 bool fPendingIntr = false;
3423 uint8_t u8Tpr = 0;
3424 uint8_t u8PendingIntr = 0;
3425 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3426 AssertRCReturn(rc, rc);
3427
3428 /*
3429 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3430 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3431 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3432 */
3433 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3434 uint32_t u32TprThreshold = 0;
3435 if (fPendingIntr)
3436 {
3437 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3438 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3439 const uint8_t u8TprPriority = u8Tpr >> 4;
3440 if (u8PendingPriority <= u8TprPriority)
3441 u32TprThreshold = u8PendingPriority;
3442 }
3443
3444 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3445 AssertRCReturn(rc, rc);
3446 }
3447
3448#ifndef IEM_VERIFICATION_MODE_FULL
3449 /* Setup the Virtualized APIC accesses. */
3450 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3451 {
3452 uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3453 if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3454 {
3455 PVM pVM = pVCpu->CTX_SUFF(pVM);
3456 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3457 RTGCPHYS GCPhysApicBase;
3458 GCPhysApicBase = u64MsrApicBase;
3459 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3460
3461 /* Unalias any existing mapping. */
3462 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3463 AssertRCReturn(rc, rc);
3464
3465 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
3466 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
3467 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3468 AssertRCReturn(rc, rc);
3469
3470 /* Update VMX's cache of the APIC base. */
3471 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3472 }
3473 }
3474#endif /* !IEM_VERIFICATION_MODE_FULL */
3475 }
3476 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3477 }
3478
3479 return rc;
3480}
3481
3482
3483/**
3484 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3485 *
3486 * @returns Guest's interruptibility-state.
3487 * @param pVCpu The cross context virtual CPU structure.
3488 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3489 * out-of-sync. Make sure to update the required fields
3490 * before using them.
3491 *
3492 * @remarks No-long-jump zone!!!
3493 */
3494DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3495{
3496 /*
3497 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3498 */
3499 uint32_t uIntrState = 0;
3500 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3501 {
3502 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3503 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3504 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3505 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3506 {
3507 if (pMixedCtx->eflags.Bits.u1IF)
3508 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3509 else
3510 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3511 }
3512 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3513 {
3514 /*
3515 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3516 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3517 */
3518 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3519 }
3520 }
3521
3522 /*
3523 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3524 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3525 * setting this would block host-NMIs and IRET will not clear the blocking.
3526 *
3527 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3528 */
3529 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3530 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3531 {
3532 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3533 }
3534
3535 return uIntrState;
3536}
3537
3538
3539/**
3540 * Loads the guest's interruptibility-state into the guest-state area in the
3541 * VMCS.
3542 *
3543 * @returns VBox status code.
3544 * @param pVCpu The cross context virtual CPU structure.
3545 * @param uIntrState The interruptibility-state to set.
3546 */
3547static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3548{
3549 NOREF(pVCpu);
3550 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3551 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3552 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3553 AssertRC(rc);
3554 return rc;
3555}
3556
3557
3558/**
3559 * Loads the exception intercepts required for guest execution in the VMCS.
3560 *
3561 * @returns VBox status code.
3562 * @param pVCpu The cross context virtual CPU structure.
3563 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3564 * out-of-sync. Make sure to update the required fields
3565 * before using them.
3566 */
3567static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3568{
3569 NOREF(pMixedCtx);
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3572 {
3573 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3574 if (pVCpu->hm.s.fGIMTrapXcptUD)
3575 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3576#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3577 else
3578 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3579#endif
3580
3581 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3582 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3583
3584 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3585 AssertRCReturn(rc, rc);
3586
3587 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3588 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3589 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3590 }
3591 return rc;
3592}
3593
3594
3595/**
3596 * Loads the guest's RIP into the guest-state area in the VMCS.
3597 *
3598 * @returns VBox status code.
3599 * @param pVCpu The cross context virtual CPU structure.
3600 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3601 * out-of-sync. Make sure to update the required fields
3602 * before using them.
3603 *
3604 * @remarks No-long-jump zone!!!
3605 */
3606static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3607{
3608 int rc = VINF_SUCCESS;
3609 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3610 {
3611 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3612 AssertRCReturn(rc, rc);
3613
3614 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3615 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3616 HMCPU_CF_VALUE(pVCpu)));
3617 }
3618 return rc;
3619}
3620
3621
3622/**
3623 * Loads the guest's RSP into the guest-state area in the VMCS.
3624 *
3625 * @returns VBox status code.
3626 * @param pVCpu The cross context virtual CPU structure.
3627 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3628 * out-of-sync. Make sure to update the required fields
3629 * before using them.
3630 *
3631 * @remarks No-long-jump zone!!!
3632 */
3633static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3634{
3635 int rc = VINF_SUCCESS;
3636 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3637 {
3638 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3639 AssertRCReturn(rc, rc);
3640
3641 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3642 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3643 }
3644 return rc;
3645}
3646
3647
3648/**
3649 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3650 *
3651 * @returns VBox status code.
3652 * @param pVCpu The cross context virtual CPU structure.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3660{
3661 int rc = VINF_SUCCESS;
3662 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3663 {
3664 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3665 Let us assert it as such and use 32-bit VMWRITE. */
3666 Assert(!(pMixedCtx->rflags.u64 >> 32));
3667 X86EFLAGS Eflags = pMixedCtx->eflags;
3668 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3669 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3670 * These will never be cleared/set, unless some other part of the VMM
3671 * code is buggy - in which case we're better of finding and fixing
3672 * those bugs than hiding them. */
3673 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3674 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3675 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3676 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3677
3678 /*
3679 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3680 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3681 */
3682 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3683 {
3684 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3685 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3686 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3687 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3688 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3689 }
3690
3691 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3692 AssertRCReturn(rc, rc);
3693
3694 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3695 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3696 }
3697 return rc;
3698}
3699
3700
3701/**
3702 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3703 *
3704 * @returns VBox status code.
3705 * @param pVCpu The cross context virtual CPU structure.
3706 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3707 * out-of-sync. Make sure to update the required fields
3708 * before using them.
3709 *
3710 * @remarks No-long-jump zone!!!
3711 */
3712DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3713{
3714 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3715 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3716 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3717 AssertRCReturn(rc, rc);
3718 return rc;
3719}
3720
3721
3722/**
3723 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3724 * CR0 is partially shared with the host and we have to consider the FPU bits.
3725 *
3726 * @returns VBox status code.
3727 * @param pVCpu The cross context virtual CPU structure.
3728 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3729 * out-of-sync. Make sure to update the required fields
3730 * before using them.
3731 *
3732 * @remarks No-long-jump zone!!!
3733 */
3734static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3735{
3736 /*
3737 * Guest CR0.
3738 * Guest FPU.
3739 */
3740 int rc = VINF_SUCCESS;
3741 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3742 {
3743 Assert(!(pMixedCtx->cr0 >> 32));
3744 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3745 PVM pVM = pVCpu->CTX_SUFF(pVM);
3746
3747 /* The guest's view (read access) of its CR0 is unblemished. */
3748 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3749 AssertRCReturn(rc, rc);
3750 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3751
3752 /* Setup VT-x's view of the guest CR0. */
3753 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3754 if (pVM->hm.s.fNestedPaging)
3755 {
3756 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3757 {
3758 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3759 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3760 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3761 }
3762 else
3763 {
3764 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3765 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3766 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3767 }
3768
3769 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3770 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3771 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3772
3773 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3774 AssertRCReturn(rc, rc);
3775 }
3776 else
3777 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3778
3779 /*
3780 * Guest FPU bits.
3781 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3782 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3783 */
3784 u32GuestCR0 |= X86_CR0_NE;
3785 bool fInterceptNM = false;
3786 if (CPUMIsGuestFPUStateActive(pVCpu))
3787 {
3788 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3789 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3790 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3791 }
3792 else
3793 {
3794 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3795 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3796 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3797 }
3798
3799 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3800 bool fInterceptMF = false;
3801 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3802 fInterceptMF = true;
3803
3804 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3805 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3806 {
3807 Assert(PDMVmmDevHeapIsEnabled(pVM));
3808 Assert(pVM->hm.s.vmx.pRealModeTSS);
3809 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3810 fInterceptNM = true;
3811 fInterceptMF = true;
3812 }
3813 else
3814 {
3815 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3816 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3817 }
3818 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3819
3820 if (fInterceptNM)
3821 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3822 else
3823 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3824
3825 if (fInterceptMF)
3826 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3827 else
3828 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3829
3830 /* Additional intercepts for debugging, define these yourself explicitly. */
3831#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3832 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3833 | RT_BIT(X86_XCPT_BP)
3834 | RT_BIT(X86_XCPT_DE)
3835 | RT_BIT(X86_XCPT_NM)
3836 | RT_BIT(X86_XCPT_TS)
3837 | RT_BIT(X86_XCPT_UD)
3838 | RT_BIT(X86_XCPT_NP)
3839 | RT_BIT(X86_XCPT_SS)
3840 | RT_BIT(X86_XCPT_GP)
3841 | RT_BIT(X86_XCPT_PF)
3842 | RT_BIT(X86_XCPT_MF)
3843 ;
3844#elif defined(HMVMX_ALWAYS_TRAP_PF)
3845 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3846#endif
3847
3848 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3849
3850 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3851 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3852 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3853 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3854 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3855 else
3856 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3857
3858 u32GuestCR0 |= uSetCR0;
3859 u32GuestCR0 &= uZapCR0;
3860 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3861
3862 /* Write VT-x's view of the guest CR0 into the VMCS. */
3863 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3866 uZapCR0));
3867
3868 /*
3869 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3870 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3871 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3872 */
3873 uint32_t u32CR0Mask = 0;
3874 u32CR0Mask = X86_CR0_PE
3875 | X86_CR0_NE
3876 | X86_CR0_WP
3877 | X86_CR0_PG
3878 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3879 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3880 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3881
3882 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3883 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3884 * and @bugref{6944}. */
3885#if 0
3886 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3887 u32CR0Mask &= ~X86_CR0_PE;
3888#endif
3889 if (pVM->hm.s.fNestedPaging)
3890 u32CR0Mask &= ~X86_CR0_WP;
3891
3892 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3893 if (fInterceptNM)
3894 {
3895 u32CR0Mask |= X86_CR0_TS
3896 | X86_CR0_MP;
3897 }
3898
3899 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3900 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3901 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3902 AssertRCReturn(rc, rc);
3903 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3904
3905 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3906 }
3907 return rc;
3908}
3909
3910
3911/**
3912 * Loads the guest control registers (CR3, CR4) into the guest-state area
3913 * in the VMCS.
3914 *
3915 * @returns VBox strict status code.
3916 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3917 * without unrestricted guest access and the VMMDev is not presently
3918 * mapped (e.g. EFI32).
3919 *
3920 * @param pVCpu The cross context virtual CPU structure.
3921 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3922 * out-of-sync. Make sure to update the required fields
3923 * before using them.
3924 *
3925 * @remarks No-long-jump zone!!!
3926 */
3927static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3928{
3929 int rc = VINF_SUCCESS;
3930 PVM pVM = pVCpu->CTX_SUFF(pVM);
3931
3932 /*
3933 * Guest CR2.
3934 * It's always loaded in the assembler code. Nothing to do here.
3935 */
3936
3937 /*
3938 * Guest CR3.
3939 */
3940 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3941 {
3942 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3943 if (pVM->hm.s.fNestedPaging)
3944 {
3945 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3946
3947 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3948 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3949 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3950 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3951
3952 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3953 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3954 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3955
3956 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3957 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3958 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3959 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3960 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3961 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3962 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3963
3964 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3965 AssertRCReturn(rc, rc);
3966 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3967
3968 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3969 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3970 {
3971 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3972 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3973 {
3974 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3975 AssertRCReturn(rc, rc);
3976 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3977 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3978 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3979 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3980 AssertRCReturn(rc, rc);
3981 }
3982
3983 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3984 have Unrestricted Execution to handle the guest when it's not using paging. */
3985 GCPhysGuestCR3 = pMixedCtx->cr3;
3986 }
3987 else
3988 {
3989 /*
3990 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3991 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3992 * EPT takes care of translating it to host-physical addresses.
3993 */
3994 RTGCPHYS GCPhys;
3995 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3996
3997 /* We obtain it here every time as the guest could have relocated this PCI region. */
3998 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3999 if (RT_SUCCESS(rc))
4000 { /* likely */ }
4001 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4002 {
4003 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4004 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4005 }
4006 else
4007 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4008
4009 GCPhysGuestCR3 = GCPhys;
4010 }
4011
4012 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4013 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4014 }
4015 else
4016 {
4017 /* Non-nested paging case, just use the hypervisor's CR3. */
4018 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4019
4020 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4021 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4022 }
4023 AssertRCReturn(rc, rc);
4024
4025 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4026 }
4027
4028 /*
4029 * Guest CR4.
4030 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4031 */
4032 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4033 {
4034 Assert(!(pMixedCtx->cr4 >> 32));
4035 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4036
4037 /* The guest's view of its CR4 is unblemished. */
4038 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4039 AssertRCReturn(rc, rc);
4040 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4041
4042 /* Setup VT-x's view of the guest CR4. */
4043 /*
4044 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4045 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4046 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4047 */
4048 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4049 {
4050 Assert(pVM->hm.s.vmx.pRealModeTSS);
4051 Assert(PDMVmmDevHeapIsEnabled(pVM));
4052 u32GuestCR4 &= ~X86_CR4_VME;
4053 }
4054
4055 if (pVM->hm.s.fNestedPaging)
4056 {
4057 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4058 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4059 {
4060 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4061 u32GuestCR4 |= X86_CR4_PSE;
4062 /* Our identity mapping is a 32-bit page directory. */
4063 u32GuestCR4 &= ~X86_CR4_PAE;
4064 }
4065 /* else use guest CR4.*/
4066 }
4067 else
4068 {
4069 /*
4070 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4071 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4072 */
4073 switch (pVCpu->hm.s.enmShadowMode)
4074 {
4075 case PGMMODE_REAL: /* Real-mode. */
4076 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4077 case PGMMODE_32_BIT: /* 32-bit paging. */
4078 {
4079 u32GuestCR4 &= ~X86_CR4_PAE;
4080 break;
4081 }
4082
4083 case PGMMODE_PAE: /* PAE paging. */
4084 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4085 {
4086 u32GuestCR4 |= X86_CR4_PAE;
4087 break;
4088 }
4089
4090 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4091 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4092#ifdef VBOX_ENABLE_64_BITS_GUESTS
4093 break;
4094#endif
4095 default:
4096 AssertFailed();
4097 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4098 }
4099 }
4100
4101 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4102 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4103 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4104 u32GuestCR4 |= uSetCR4;
4105 u32GuestCR4 &= uZapCR4;
4106
4107 /* Write VT-x's view of the guest CR4 into the VMCS. */
4108 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4109 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4110 AssertRCReturn(rc, rc);
4111
4112 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4113 uint32_t u32CR4Mask = X86_CR4_VME
4114 | X86_CR4_PAE
4115 | X86_CR4_PGE
4116 | X86_CR4_PSE
4117 | X86_CR4_VMXE;
4118 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4119 u32CR4Mask |= X86_CR4_OSXSAVE;
4120 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4121 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4122 AssertRCReturn(rc, rc);
4123
4124 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4125 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4126
4127 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4128 }
4129 return rc;
4130}
4131
4132
4133/**
4134 * Loads the guest debug registers into the guest-state area in the VMCS.
4135 *
4136 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4137 *
4138 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4139 *
4140 * @returns VBox status code.
4141 * @param pVCpu The cross context virtual CPU structure.
4142 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4143 * out-of-sync. Make sure to update the required fields
4144 * before using them.
4145 *
4146 * @remarks No-long-jump zone!!!
4147 */
4148static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4149{
4150 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4151 return VINF_SUCCESS;
4152
4153#ifdef VBOX_STRICT
4154 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4155 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4156 {
4157 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4158 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4159 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4160 }
4161#endif
4162
4163 int rc;
4164 PVM pVM = pVCpu->CTX_SUFF(pVM);
4165 bool fSteppingDB = false;
4166 bool fInterceptMovDRx = false;
4167 if (pVCpu->hm.s.fSingleInstruction)
4168 {
4169 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4170 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4171 {
4172 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4173 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4174 AssertRCReturn(rc, rc);
4175 Assert(fSteppingDB == false);
4176 }
4177 else
4178 {
4179 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4180 pVCpu->hm.s.fClearTrapFlag = true;
4181 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4182 fSteppingDB = true;
4183 }
4184 }
4185
4186 if ( fSteppingDB
4187 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4188 {
4189 /*
4190 * Use the combined guest and host DRx values found in the hypervisor
4191 * register set because the debugger has breakpoints active or someone
4192 * is single stepping on the host side without a monitor trap flag.
4193 *
4194 * Note! DBGF expects a clean DR6 state before executing guest code.
4195 */
4196#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4197 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4198 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4199 {
4200 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4201 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4202 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4203 }
4204 else
4205#endif
4206 if (!CPUMIsHyperDebugStateActive(pVCpu))
4207 {
4208 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4209 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4210 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4211 }
4212
4213 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4214 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4215 AssertRCReturn(rc, rc);
4216
4217 pVCpu->hm.s.fUsingHyperDR7 = true;
4218 fInterceptMovDRx = true;
4219 }
4220 else
4221 {
4222 /*
4223 * If the guest has enabled debug registers, we need to load them prior to
4224 * executing guest code so they'll trigger at the right time.
4225 */
4226 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4227 {
4228#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4229 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4230 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4231 {
4232 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4233 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4234 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4235 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4236 }
4237 else
4238#endif
4239 if (!CPUMIsGuestDebugStateActive(pVCpu))
4240 {
4241 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4242 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4243 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4244 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4245 }
4246 Assert(!fInterceptMovDRx);
4247 }
4248 /*
4249 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4250 * must intercept #DB in order to maintain a correct DR6 guest value, and
4251 * because we need to intercept it to prevent nested #DBs from hanging the
4252 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4253 */
4254#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4255 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4256 && !CPUMIsGuestDebugStateActive(pVCpu))
4257#else
4258 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4259#endif
4260 {
4261 fInterceptMovDRx = true;
4262 }
4263
4264 /* Update guest DR7. */
4265 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4266 AssertRCReturn(rc, rc);
4267
4268 pVCpu->hm.s.fUsingHyperDR7 = false;
4269 }
4270
4271 /*
4272 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4273 */
4274 if (fInterceptMovDRx)
4275 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4276 else
4277 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4278 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4279 AssertRCReturn(rc, rc);
4280
4281 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4282 return VINF_SUCCESS;
4283}
4284
4285
4286#ifdef VBOX_STRICT
4287/**
4288 * Strict function to validate segment registers.
4289 *
4290 * @remarks ASSUMES CR0 is up to date.
4291 */
4292static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4293{
4294 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4295 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4296 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4297 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4298 && ( !CPUMIsGuestInRealModeEx(pCtx)
4299 && !CPUMIsGuestInV86ModeEx(pCtx)))
4300 {
4301 /* Protected mode checks */
4302 /* CS */
4303 Assert(pCtx->cs.Attr.n.u1Present);
4304 Assert(!(pCtx->cs.Attr.u & 0xf00));
4305 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4306 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4307 || !(pCtx->cs.Attr.n.u1Granularity));
4308 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4309 || (pCtx->cs.Attr.n.u1Granularity));
4310 /* CS cannot be loaded with NULL in protected mode. */
4311 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4312 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4313 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4314 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4315 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4316 else
4317 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4318 /* SS */
4319 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4320 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4321 if ( !(pCtx->cr0 & X86_CR0_PE)
4322 || pCtx->cs.Attr.n.u4Type == 3)
4323 {
4324 Assert(!pCtx->ss.Attr.n.u2Dpl);
4325 }
4326 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4327 {
4328 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4329 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4330 Assert(pCtx->ss.Attr.n.u1Present);
4331 Assert(!(pCtx->ss.Attr.u & 0xf00));
4332 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4333 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4334 || !(pCtx->ss.Attr.n.u1Granularity));
4335 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4336 || (pCtx->ss.Attr.n.u1Granularity));
4337 }
4338 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4339 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4340 {
4341 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4342 Assert(pCtx->ds.Attr.n.u1Present);
4343 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4344 Assert(!(pCtx->ds.Attr.u & 0xf00));
4345 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4346 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4347 || !(pCtx->ds.Attr.n.u1Granularity));
4348 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4349 || (pCtx->ds.Attr.n.u1Granularity));
4350 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4351 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4352 }
4353 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4354 {
4355 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4356 Assert(pCtx->es.Attr.n.u1Present);
4357 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4358 Assert(!(pCtx->es.Attr.u & 0xf00));
4359 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4360 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4361 || !(pCtx->es.Attr.n.u1Granularity));
4362 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4363 || (pCtx->es.Attr.n.u1Granularity));
4364 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4365 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4366 }
4367 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4368 {
4369 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4370 Assert(pCtx->fs.Attr.n.u1Present);
4371 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4372 Assert(!(pCtx->fs.Attr.u & 0xf00));
4373 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4374 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4375 || !(pCtx->fs.Attr.n.u1Granularity));
4376 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4377 || (pCtx->fs.Attr.n.u1Granularity));
4378 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4379 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4380 }
4381 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4382 {
4383 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4384 Assert(pCtx->gs.Attr.n.u1Present);
4385 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4386 Assert(!(pCtx->gs.Attr.u & 0xf00));
4387 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4388 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4389 || !(pCtx->gs.Attr.n.u1Granularity));
4390 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4391 || (pCtx->gs.Attr.n.u1Granularity));
4392 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4393 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4394 }
4395 /* 64-bit capable CPUs. */
4396# if HC_ARCH_BITS == 64
4397 Assert(!(pCtx->cs.u64Base >> 32));
4398 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4399 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4400 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4401# endif
4402 }
4403 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4404 || ( CPUMIsGuestInRealModeEx(pCtx)
4405 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4406 {
4407 /* Real and v86 mode checks. */
4408 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4409 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4410 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4411 {
4412 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4413 }
4414 else
4415 {
4416 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4417 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4418 }
4419
4420 /* CS */
4421 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4422 Assert(pCtx->cs.u32Limit == 0xffff);
4423 Assert(u32CSAttr == 0xf3);
4424 /* SS */
4425 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4426 Assert(pCtx->ss.u32Limit == 0xffff);
4427 Assert(u32SSAttr == 0xf3);
4428 /* DS */
4429 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4430 Assert(pCtx->ds.u32Limit == 0xffff);
4431 Assert(u32DSAttr == 0xf3);
4432 /* ES */
4433 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4434 Assert(pCtx->es.u32Limit == 0xffff);
4435 Assert(u32ESAttr == 0xf3);
4436 /* FS */
4437 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4438 Assert(pCtx->fs.u32Limit == 0xffff);
4439 Assert(u32FSAttr == 0xf3);
4440 /* GS */
4441 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4442 Assert(pCtx->gs.u32Limit == 0xffff);
4443 Assert(u32GSAttr == 0xf3);
4444 /* 64-bit capable CPUs. */
4445# if HC_ARCH_BITS == 64
4446 Assert(!(pCtx->cs.u64Base >> 32));
4447 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4448 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4449 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4450# endif
4451 }
4452}
4453#endif /* VBOX_STRICT */
4454
4455
4456/**
4457 * Writes a guest segment register into the guest-state area in the VMCS.
4458 *
4459 * @returns VBox status code.
4460 * @param pVCpu The cross context virtual CPU structure.
4461 * @param idxSel Index of the selector in the VMCS.
4462 * @param idxLimit Index of the segment limit in the VMCS.
4463 * @param idxBase Index of the segment base in the VMCS.
4464 * @param idxAccess Index of the access rights of the segment in the VMCS.
4465 * @param pSelReg Pointer to the segment selector.
4466 *
4467 * @remarks No-long-jump zone!!!
4468 */
4469static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4470 uint32_t idxAccess, PCPUMSELREG pSelReg)
4471{
4472 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4473 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4474 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4475 AssertRCReturn(rc, rc);
4476
4477 uint32_t u32Access = pSelReg->Attr.u;
4478 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479 {
4480 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4481 u32Access = 0xf3;
4482 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4483 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4484 }
4485 else
4486 {
4487 /*
4488 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4489 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4490 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4491 * loaded in protected-mode have their attribute as 0.
4492 */
4493 if (!u32Access)
4494 u32Access = X86DESCATTR_UNUSABLE;
4495 }
4496
4497 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4498 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4499 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4500
4501 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4502 AssertRCReturn(rc, rc);
4503 return rc;
4504}
4505
4506
4507/**
4508 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4509 * into the guest-state area in the VMCS.
4510 *
4511 * @returns VBox status code.
4512 * @param pVCpu The cross context virtual CPU structure.
4513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4514 * out-of-sync. Make sure to update the required fields
4515 * before using them.
4516 *
4517 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4518 * @remarks No-long-jump zone!!!
4519 */
4520static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4521{
4522 int rc = VERR_INTERNAL_ERROR_5;
4523 PVM pVM = pVCpu->CTX_SUFF(pVM);
4524
4525 /*
4526 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4527 */
4528 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4529 {
4530 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4531 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4532 {
4533 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4534 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4535 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4536 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4537 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4538 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4539 }
4540
4541#ifdef VBOX_WITH_REM
4542 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4543 {
4544 Assert(pVM->hm.s.vmx.pRealModeTSS);
4545 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4546 if ( pVCpu->hm.s.vmx.fWasInRealMode
4547 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4548 {
4549 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4550 in real-mode (e.g. OpenBSD 4.0) */
4551 REMFlushTBs(pVM);
4552 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4553 pVCpu->hm.s.vmx.fWasInRealMode = false;
4554 }
4555 }
4556#endif
4557 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4558 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4559 AssertRCReturn(rc, rc);
4560 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4561 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4562 AssertRCReturn(rc, rc);
4563 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4564 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4565 AssertRCReturn(rc, rc);
4566 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4567 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4568 AssertRCReturn(rc, rc);
4569 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4570 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4571 AssertRCReturn(rc, rc);
4572 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4573 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4574 AssertRCReturn(rc, rc);
4575
4576#ifdef VBOX_STRICT
4577 /* Validate. */
4578 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4579#endif
4580
4581 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4582 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4583 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4584 }
4585
4586 /*
4587 * Guest TR.
4588 */
4589 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4590 {
4591 /*
4592 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4593 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4594 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4595 */
4596 uint16_t u16Sel = 0;
4597 uint32_t u32Limit = 0;
4598 uint64_t u64Base = 0;
4599 uint32_t u32AccessRights = 0;
4600
4601 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4602 {
4603 u16Sel = pMixedCtx->tr.Sel;
4604 u32Limit = pMixedCtx->tr.u32Limit;
4605 u64Base = pMixedCtx->tr.u64Base;
4606 u32AccessRights = pMixedCtx->tr.Attr.u;
4607 }
4608 else
4609 {
4610 Assert(pVM->hm.s.vmx.pRealModeTSS);
4611 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4612
4613 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4614 RTGCPHYS GCPhys;
4615 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4616 AssertRCReturn(rc, rc);
4617
4618 X86DESCATTR DescAttr;
4619 DescAttr.u = 0;
4620 DescAttr.n.u1Present = 1;
4621 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4622
4623 u16Sel = 0;
4624 u32Limit = HM_VTX_TSS_SIZE;
4625 u64Base = GCPhys; /* in real-mode phys = virt. */
4626 u32AccessRights = DescAttr.u;
4627 }
4628
4629 /* Validate. */
4630 Assert(!(u16Sel & RT_BIT(2)));
4631 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4632 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4633 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4634 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4635 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4636 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4637 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4638 Assert( (u32Limit & 0xfff) == 0xfff
4639 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4640 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4641 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4647 AssertRCReturn(rc, rc);
4648
4649 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4650 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4651 }
4652
4653 /*
4654 * Guest GDTR.
4655 */
4656 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4657 {
4658 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4659 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4660 AssertRCReturn(rc, rc);
4661
4662 /* Validate. */
4663 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4664
4665 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4666 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4667 }
4668
4669 /*
4670 * Guest LDTR.
4671 */
4672 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4673 {
4674 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4675 uint32_t u32Access = 0;
4676 if (!pMixedCtx->ldtr.Attr.u)
4677 u32Access = X86DESCATTR_UNUSABLE;
4678 else
4679 u32Access = pMixedCtx->ldtr.Attr.u;
4680
4681 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4682 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4683 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4684 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4685 AssertRCReturn(rc, rc);
4686
4687 /* Validate. */
4688 if (!(u32Access & X86DESCATTR_UNUSABLE))
4689 {
4690 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4691 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4692 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4693 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4694 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4695 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4696 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4697 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4698 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4699 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4700 }
4701
4702 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4703 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4704 }
4705
4706 /*
4707 * Guest IDTR.
4708 */
4709 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4710 {
4711 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4712 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4713 AssertRCReturn(rc, rc);
4714
4715 /* Validate. */
4716 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4717
4718 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4719 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4720 }
4721
4722 return VINF_SUCCESS;
4723}
4724
4725
4726/**
4727 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4728 * areas.
4729 *
4730 * These MSRs will automatically be loaded to the host CPU on every successful
4731 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4732 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4733 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4734 *
4735 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4736 *
4737 * @returns VBox status code.
4738 * @param pVCpu The cross context virtual CPU structure.
4739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4740 * out-of-sync. Make sure to update the required fields
4741 * before using them.
4742 *
4743 * @remarks No-long-jump zone!!!
4744 */
4745static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4746{
4747 AssertPtr(pVCpu);
4748 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4749
4750 /*
4751 * MSRs that we use the auto-load/store MSR area in the VMCS.
4752 */
4753 PVM pVM = pVCpu->CTX_SUFF(pVM);
4754 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4755 {
4756 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4757#if HC_ARCH_BITS == 32
4758 if (pVM->hm.s.fAllow64BitGuests)
4759 {
4760 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4761 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4762 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4763 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4764 AssertRCReturn(rc, rc);
4765# ifdef LOG_ENABLED
4766 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4767 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4768 {
4769 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4770 pMsr->u64Value));
4771 }
4772# endif
4773 }
4774#endif
4775 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4776 }
4777
4778 /*
4779 * Guest Sysenter MSRs.
4780 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4781 * VM-exits on WRMSRs for these MSRs.
4782 */
4783 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4784 {
4785 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4787 }
4788
4789 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4790 {
4791 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4793 }
4794
4795 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4796 {
4797 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4798 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4799 }
4800
4801 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4802 {
4803 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4804 {
4805 /*
4806 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4807 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4808 */
4809 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4810 {
4811 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4812 AssertRCReturn(rc,rc);
4813 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4814 }
4815 else
4816 {
4817 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4818 NULL /* pfAddedAndUpdated */);
4819 AssertRCReturn(rc, rc);
4820
4821 /* We need to intercept reads too, see @bugref{7386#c16}. */
4822 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4823 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4824 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4825 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4826 }
4827 }
4828 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4829 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4830 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4831 }
4832
4833 return VINF_SUCCESS;
4834}
4835
4836
4837/**
4838 * Loads the guest activity state into the guest-state area in the VMCS.
4839 *
4840 * @returns VBox status code.
4841 * @param pVCpu The cross context virtual CPU structure.
4842 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4843 * out-of-sync. Make sure to update the required fields
4844 * before using them.
4845 *
4846 * @remarks No-long-jump zone!!!
4847 */
4848static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4849{
4850 NOREF(pMixedCtx);
4851 /** @todo See if we can make use of other states, e.g.
4852 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4853 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4854 {
4855 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4856 AssertRCReturn(rc, rc);
4857
4858 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4859 }
4860 return VINF_SUCCESS;
4861}
4862
4863
4864#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4865/**
4866 * Check if guest state allows safe use of 32-bit switcher again.
4867 *
4868 * Segment bases and protected mode structures must be 32-bit addressable
4869 * because the 32-bit switcher will ignore high dword when writing these VMCS
4870 * fields. See @bugref{8432} for details.
4871 *
4872 * @returns true if safe, false if must continue to use the 64-bit switcher.
4873 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4874 * out-of-sync. Make sure to update the required fields
4875 * before using them.
4876 *
4877 * @remarks No-long-jump zone!!!
4878 */
4879static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4880{
4881 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4882 return false;
4883 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4884 return false;
4885 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4886 return false;
4887 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4888 return false;
4889 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4890 return false;
4891 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4892 return false;
4893 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4894 return false;
4895 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4896 return false;
4897 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4898 return false;
4899 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4900 return false;
4901 /* All good, bases are 32-bit. */
4902 return true;
4903}
4904#endif
4905
4906
4907/**
4908 * Sets up the appropriate function to run guest code.
4909 *
4910 * @returns VBox status code.
4911 * @param pVCpu The cross context virtual CPU structure.
4912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4913 * out-of-sync. Make sure to update the required fields
4914 * before using them.
4915 *
4916 * @remarks No-long-jump zone!!!
4917 */
4918static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4919{
4920 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4921 {
4922#ifndef VBOX_ENABLE_64_BITS_GUESTS
4923 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4924#endif
4925 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4926#if HC_ARCH_BITS == 32
4927 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4928 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4929 {
4930 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4931 {
4932 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4933 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4934 | HM_CHANGED_VMX_ENTRY_CTLS
4935 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4936 }
4937 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4938
4939 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4940 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4941 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4942 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4943 }
4944#else
4945 /* 64-bit host. */
4946 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4947#endif
4948 }
4949 else
4950 {
4951 /* Guest is not in long mode, use the 32-bit handler. */
4952#if HC_ARCH_BITS == 32
4953 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4954 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4955 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4956 {
4957 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4958 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4959 | HM_CHANGED_VMX_ENTRY_CTLS
4960 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4961 }
4962# ifdef VBOX_ENABLE_64_BITS_GUESTS
4963 /*
4964 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4965 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4966 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4967 * the much faster 32-bit switcher again.
4968 */
4969 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4970 {
4971 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4972 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4973 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4974 }
4975 else
4976 {
4977 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4978 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4979 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4980 {
4981 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4982 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4983 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4984 | HM_CHANGED_VMX_ENTRY_CTLS
4985 | HM_CHANGED_VMX_EXIT_CTLS
4986 | HM_CHANGED_HOST_CONTEXT);
4987 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4988 }
4989 }
4990# else
4991 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4992# endif
4993#else
4994 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4995#endif
4996 }
4997 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4998 return VINF_SUCCESS;
4999}
5000
5001
5002/**
5003 * Wrapper for running the guest code in VT-x.
5004 *
5005 * @returns VBox status code, no informational status codes.
5006 * @param pVM The cross context VM structure.
5007 * @param pVCpu The cross context virtual CPU structure.
5008 * @param pCtx Pointer to the guest-CPU context.
5009 *
5010 * @remarks No-long-jump zone!!!
5011 */
5012DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5013{
5014 /*
5015 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5016 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5017 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5018 */
5019 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5020 /** @todo Add stats for resume vs launch. */
5021#ifdef VBOX_WITH_KERNEL_USING_XMM
5022 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5023#else
5024 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5025#endif
5026 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5027 return rc;
5028}
5029
5030
5031/**
5032 * Reports world-switch error and dumps some useful debug info.
5033 *
5034 * @param pVM The cross context VM structure.
5035 * @param pVCpu The cross context virtual CPU structure.
5036 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5037 * @param pCtx Pointer to the guest-CPU context.
5038 * @param pVmxTransient Pointer to the VMX transient structure (only
5039 * exitReason updated).
5040 */
5041static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5042{
5043 Assert(pVM);
5044 Assert(pVCpu);
5045 Assert(pCtx);
5046 Assert(pVmxTransient);
5047 HMVMX_ASSERT_PREEMPT_SAFE();
5048
5049 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5050 switch (rcVMRun)
5051 {
5052 case VERR_VMX_INVALID_VMXON_PTR:
5053 AssertFailed();
5054 break;
5055 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5056 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5057 {
5058 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5059 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5060 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5061 AssertRC(rc);
5062
5063 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5064 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5065 Cannot do it here as we may have been long preempted. */
5066
5067#ifdef VBOX_STRICT
5068 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5069 pVmxTransient->uExitReason));
5070 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5071 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5072 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5073 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5074 else
5075 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5076 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5077 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5078
5079 /* VMX control bits. */
5080 uint32_t u32Val;
5081 uint64_t u64Val;
5082 RTHCUINTREG uHCReg;
5083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5085 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5086 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5087 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5088 {
5089 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5090 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5091 }
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5107 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5113 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5119 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5121 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5122 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5123 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5124 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5125 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5126 if (pVM->hm.s.fNestedPaging)
5127 {
5128 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5129 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5130 }
5131
5132 /* Guest bits. */
5133 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5134 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5135 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5136 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5137 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5138 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5139 if (pVM->hm.s.vmx.fVpid)
5140 {
5141 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5142 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5143 }
5144
5145 /* Host bits. */
5146 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5147 Log4(("Host CR0 %#RHr\n", uHCReg));
5148 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5149 Log4(("Host CR3 %#RHr\n", uHCReg));
5150 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5151 Log4(("Host CR4 %#RHr\n", uHCReg));
5152
5153 RTGDTR HostGdtr;
5154 PCX86DESCHC pDesc;
5155 ASMGetGDTR(&HostGdtr);
5156 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5157 Log4(("Host CS %#08x\n", u32Val));
5158 if (u32Val < HostGdtr.cbGdt)
5159 {
5160 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5161 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5162 }
5163
5164 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5165 Log4(("Host DS %#08x\n", u32Val));
5166 if (u32Val < HostGdtr.cbGdt)
5167 {
5168 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5169 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5170 }
5171
5172 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5173 Log4(("Host ES %#08x\n", u32Val));
5174 if (u32Val < HostGdtr.cbGdt)
5175 {
5176 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5177 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5178 }
5179
5180 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5181 Log4(("Host FS %#08x\n", u32Val));
5182 if (u32Val < HostGdtr.cbGdt)
5183 {
5184 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5185 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5186 }
5187
5188 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5189 Log4(("Host GS %#08x\n", u32Val));
5190 if (u32Val < HostGdtr.cbGdt)
5191 {
5192 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5193 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5194 }
5195
5196 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5197 Log4(("Host SS %#08x\n", u32Val));
5198 if (u32Val < HostGdtr.cbGdt)
5199 {
5200 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5201 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5202 }
5203
5204 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5205 Log4(("Host TR %#08x\n", u32Val));
5206 if (u32Val < HostGdtr.cbGdt)
5207 {
5208 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5209 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5210 }
5211
5212 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5213 Log4(("Host TR Base %#RHv\n", uHCReg));
5214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5215 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5216 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5217 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5218 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5219 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5220 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5221 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5222 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5223 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5224 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5225 Log4(("Host RSP %#RHv\n", uHCReg));
5226 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5227 Log4(("Host RIP %#RHv\n", uHCReg));
5228# if HC_ARCH_BITS == 64
5229 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5230 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5231 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5232 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5233 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5234 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5235# endif
5236#endif /* VBOX_STRICT */
5237 break;
5238 }
5239
5240 default:
5241 /* Impossible */
5242 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5243 break;
5244 }
5245 NOREF(pVM); NOREF(pCtx);
5246}
5247
5248
5249#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5250#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5251# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5252#endif
5253#ifdef VBOX_STRICT
5254static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5255{
5256 switch (idxField)
5257 {
5258 case VMX_VMCS_GUEST_RIP:
5259 case VMX_VMCS_GUEST_RSP:
5260 case VMX_VMCS_GUEST_SYSENTER_EIP:
5261 case VMX_VMCS_GUEST_SYSENTER_ESP:
5262 case VMX_VMCS_GUEST_GDTR_BASE:
5263 case VMX_VMCS_GUEST_IDTR_BASE:
5264 case VMX_VMCS_GUEST_CS_BASE:
5265 case VMX_VMCS_GUEST_DS_BASE:
5266 case VMX_VMCS_GUEST_ES_BASE:
5267 case VMX_VMCS_GUEST_FS_BASE:
5268 case VMX_VMCS_GUEST_GS_BASE:
5269 case VMX_VMCS_GUEST_SS_BASE:
5270 case VMX_VMCS_GUEST_LDTR_BASE:
5271 case VMX_VMCS_GUEST_TR_BASE:
5272 case VMX_VMCS_GUEST_CR3:
5273 return true;
5274 }
5275 return false;
5276}
5277
5278static bool hmR0VmxIsValidReadField(uint32_t idxField)
5279{
5280 switch (idxField)
5281 {
5282 /* Read-only fields. */
5283 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5284 return true;
5285 }
5286 /* Remaining readable fields should also be writable. */
5287 return hmR0VmxIsValidWriteField(idxField);
5288}
5289#endif /* VBOX_STRICT */
5290
5291
5292/**
5293 * Executes the specified handler in 64-bit mode.
5294 *
5295 * @returns VBox status code (no informational status codes).
5296 * @param pVM The cross context VM structure.
5297 * @param pVCpu The cross context virtual CPU structure.
5298 * @param pCtx Pointer to the guest CPU context.
5299 * @param enmOp The operation to perform.
5300 * @param cParams Number of parameters.
5301 * @param paParam Array of 32-bit parameters.
5302 */
5303VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5304 uint32_t cParams, uint32_t *paParam)
5305{
5306 NOREF(pCtx);
5307
5308 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5309 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5310 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5311 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5312
5313#ifdef VBOX_STRICT
5314 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5315 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5316
5317 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5318 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5319#endif
5320
5321 /* Disable interrupts. */
5322 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5323
5324#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5325 RTCPUID idHostCpu = RTMpCpuId();
5326 CPUMR0SetLApic(pVCpu, idHostCpu);
5327#endif
5328
5329 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5330 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5331
5332 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5333 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5334 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5335
5336 /* Leave VMX Root Mode. */
5337 VMXDisable();
5338
5339 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5340
5341 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5342 CPUMSetHyperEIP(pVCpu, enmOp);
5343 for (int i = (int)cParams - 1; i >= 0; i--)
5344 CPUMPushHyper(pVCpu, paParam[i]);
5345
5346 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5347
5348 /* Call the switcher. */
5349 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5350 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5351
5352 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5353 /* Make sure the VMX instructions don't cause #UD faults. */
5354 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5355
5356 /* Re-enter VMX Root Mode */
5357 int rc2 = VMXEnable(HCPhysCpuPage);
5358 if (RT_FAILURE(rc2))
5359 {
5360 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5361 ASMSetFlags(fOldEFlags);
5362 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5363 return rc2;
5364 }
5365
5366 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5367 AssertRC(rc2);
5368 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5369 Assert(!(ASMGetFlags() & X86_EFL_IF));
5370 ASMSetFlags(fOldEFlags);
5371 return rc;
5372}
5373
5374
5375/**
5376 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5377 * supporting 64-bit guests.
5378 *
5379 * @returns VBox status code.
5380 * @param fResume Whether to VMLAUNCH or VMRESUME.
5381 * @param pCtx Pointer to the guest-CPU context.
5382 * @param pCache Pointer to the VMCS cache.
5383 * @param pVM The cross context VM structure.
5384 * @param pVCpu The cross context virtual CPU structure.
5385 */
5386DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5387{
5388 NOREF(fResume);
5389
5390 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5391 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5392
5393#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5394 pCache->uPos = 1;
5395 pCache->interPD = PGMGetInterPaeCR3(pVM);
5396 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5397#endif
5398
5399#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5400 pCache->TestIn.HCPhysCpuPage = 0;
5401 pCache->TestIn.HCPhysVmcs = 0;
5402 pCache->TestIn.pCache = 0;
5403 pCache->TestOut.HCPhysVmcs = 0;
5404 pCache->TestOut.pCache = 0;
5405 pCache->TestOut.pCtx = 0;
5406 pCache->TestOut.eflags = 0;
5407#else
5408 NOREF(pCache);
5409#endif
5410
5411 uint32_t aParam[10];
5412 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5413 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5414 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5415 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5416 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5417 aParam[5] = 0;
5418 aParam[6] = VM_RC_ADDR(pVM, pVM);
5419 aParam[7] = 0;
5420 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5421 aParam[9] = 0;
5422
5423#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5424 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5425 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5426#endif
5427 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5428
5429#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5430 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5431 Assert(pCtx->dr[4] == 10);
5432 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5433#endif
5434
5435#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5436 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5437 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5438 pVCpu->hm.s.vmx.HCPhysVmcs));
5439 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5440 pCache->TestOut.HCPhysVmcs));
5441 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5442 pCache->TestOut.pCache));
5443 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5444 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5445 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5446 pCache->TestOut.pCtx));
5447 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5448#endif
5449 return rc;
5450}
5451
5452
5453/**
5454 * Initialize the VMCS-Read cache.
5455 *
5456 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5457 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5458 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5459 * (those that have a 32-bit FULL & HIGH part).
5460 *
5461 * @returns VBox status code.
5462 * @param pVM The cross context VM structure.
5463 * @param pVCpu The cross context virtual CPU structure.
5464 */
5465static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5466{
5467#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5468{ \
5469 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5470 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5471 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5472 ++cReadFields; \
5473}
5474
5475 AssertPtr(pVM);
5476 AssertPtr(pVCpu);
5477 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5478 uint32_t cReadFields = 0;
5479
5480 /*
5481 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5482 * and serve to indicate exceptions to the rules.
5483 */
5484
5485 /* Guest-natural selector base fields. */
5486#if 0
5487 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5490#endif
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5499 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5502 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5503#if 0
5504 /* Unused natural width guest-state fields. */
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5507#endif
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5510
5511 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5512#if 0
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5522#endif
5523
5524 /* Natural width guest-state fields. */
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5526#if 0
5527 /* Currently unused field. */
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5529#endif
5530
5531 if (pVM->hm.s.fNestedPaging)
5532 {
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5534 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5535 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5536 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5537 }
5538 else
5539 {
5540 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5541 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5542 }
5543
5544#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5545 return VINF_SUCCESS;
5546}
5547
5548
5549/**
5550 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5551 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5552 * darwin, running 64-bit guests).
5553 *
5554 * @returns VBox status code.
5555 * @param pVCpu The cross context virtual CPU structure.
5556 * @param idxField The VMCS field encoding.
5557 * @param u64Val 16, 32 or 64-bit value.
5558 */
5559VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5560{
5561 int rc;
5562 switch (idxField)
5563 {
5564 /*
5565 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5566 */
5567 /* 64-bit Control fields. */
5568 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5569 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5570 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5571 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5572 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5573 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5574 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5575 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5576 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5577 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5578 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5579 case VMX_VMCS64_CTRL_EPTP_FULL:
5580 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5581 /* 64-bit Guest-state fields. */
5582 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5583 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5584 case VMX_VMCS64_GUEST_PAT_FULL:
5585 case VMX_VMCS64_GUEST_EFER_FULL:
5586 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5587 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5588 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5589 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5590 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5591 /* 64-bit Host-state fields. */
5592 case VMX_VMCS64_HOST_PAT_FULL:
5593 case VMX_VMCS64_HOST_EFER_FULL:
5594 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5595 {
5596 rc = VMXWriteVmcs32(idxField, u64Val);
5597 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5598 break;
5599 }
5600
5601 /*
5602 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5603 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5604 */
5605 /* Natural-width Guest-state fields. */
5606 case VMX_VMCS_GUEST_CR3:
5607 case VMX_VMCS_GUEST_ES_BASE:
5608 case VMX_VMCS_GUEST_CS_BASE:
5609 case VMX_VMCS_GUEST_SS_BASE:
5610 case VMX_VMCS_GUEST_DS_BASE:
5611 case VMX_VMCS_GUEST_FS_BASE:
5612 case VMX_VMCS_GUEST_GS_BASE:
5613 case VMX_VMCS_GUEST_LDTR_BASE:
5614 case VMX_VMCS_GUEST_TR_BASE:
5615 case VMX_VMCS_GUEST_GDTR_BASE:
5616 case VMX_VMCS_GUEST_IDTR_BASE:
5617 case VMX_VMCS_GUEST_RSP:
5618 case VMX_VMCS_GUEST_RIP:
5619 case VMX_VMCS_GUEST_SYSENTER_ESP:
5620 case VMX_VMCS_GUEST_SYSENTER_EIP:
5621 {
5622 if (!(u64Val >> 32))
5623 {
5624 /* If this field is 64-bit, VT-x will zero out the top bits. */
5625 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5626 }
5627 else
5628 {
5629 /* Assert that only the 32->64 switcher case should ever come here. */
5630 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5631 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5632 }
5633 break;
5634 }
5635
5636 default:
5637 {
5638 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5639 rc = VERR_INVALID_PARAMETER;
5640 break;
5641 }
5642 }
5643 AssertRCReturn(rc, rc);
5644 return rc;
5645}
5646
5647
5648/**
5649 * Queue up a VMWRITE by using the VMCS write cache.
5650 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5651 *
5652 * @param pVCpu The cross context virtual CPU structure.
5653 * @param idxField The VMCS field encoding.
5654 * @param u64Val 16, 32 or 64-bit value.
5655 */
5656VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5657{
5658 AssertPtr(pVCpu);
5659 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5660
5661 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5662 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5663
5664 /* Make sure there are no duplicates. */
5665 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5666 {
5667 if (pCache->Write.aField[i] == idxField)
5668 {
5669 pCache->Write.aFieldVal[i] = u64Val;
5670 return VINF_SUCCESS;
5671 }
5672 }
5673
5674 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5675 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5676 pCache->Write.cValidEntries++;
5677 return VINF_SUCCESS;
5678}
5679#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5680
5681
5682/**
5683 * Sets up the usage of TSC-offsetting and updates the VMCS.
5684 *
5685 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5686 * VMX preemption timer.
5687 *
5688 * @returns VBox status code.
5689 * @param pVM The cross context VM structure.
5690 * @param pVCpu The cross context virtual CPU structure.
5691 *
5692 * @remarks No-long-jump zone!!!
5693 */
5694static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5695{
5696 int rc;
5697 bool fOffsettedTsc;
5698 bool fParavirtTsc;
5699 if (pVM->hm.s.vmx.fUsePreemptTimer)
5700 {
5701 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5702 &fOffsettedTsc, &fParavirtTsc);
5703
5704 /* Make sure the returned values have sane upper and lower boundaries. */
5705 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5706 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5707 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5708 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5709
5710 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5711 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5712 }
5713 else
5714 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5715
5716 /** @todo later optimize this to be done elsewhere and not before every
5717 * VM-entry. */
5718 if (fParavirtTsc)
5719 {
5720 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5721 information before every VM-entry, hence disable it for performance sake. */
5722#if 0
5723 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5724 AssertRC(rc);
5725#endif
5726 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5727 }
5728
5729 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5730 {
5731 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5732 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5733
5734 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5736 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5737 }
5738 else
5739 {
5740 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5741 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5742 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5743 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5744 }
5745}
5746
5747
5748/**
5749 * Determines if an exception is a contributory exception.
5750 *
5751 * Contributory exceptions are ones which can cause double-faults unless the
5752 * original exception was a benign exception. Page-fault is intentionally not
5753 * included here as it's a conditional contributory exception.
5754 *
5755 * @returns true if the exception is contributory, false otherwise.
5756 * @param uVector The exception vector.
5757 */
5758DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5759{
5760 switch (uVector)
5761 {
5762 case X86_XCPT_GP:
5763 case X86_XCPT_SS:
5764 case X86_XCPT_NP:
5765 case X86_XCPT_TS:
5766 case X86_XCPT_DE:
5767 return true;
5768 default:
5769 break;
5770 }
5771 return false;
5772}
5773
5774
5775/**
5776 * Sets an event as a pending event to be injected into the guest.
5777 *
5778 * @param pVCpu The cross context virtual CPU structure.
5779 * @param u32IntInfo The VM-entry interruption-information field.
5780 * @param cbInstr The VM-entry instruction length in bytes (for software
5781 * interrupts, exceptions and privileged software
5782 * exceptions).
5783 * @param u32ErrCode The VM-entry exception error code.
5784 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5785 * page-fault.
5786 *
5787 * @remarks Statistics counter assumes this is a guest event being injected or
5788 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5789 * always incremented.
5790 */
5791DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5792 RTGCUINTPTR GCPtrFaultAddress)
5793{
5794 Assert(!pVCpu->hm.s.Event.fPending);
5795 pVCpu->hm.s.Event.fPending = true;
5796 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5797 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5798 pVCpu->hm.s.Event.cbInstr = cbInstr;
5799 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5800}
5801
5802
5803/**
5804 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5805 *
5806 * @param pVCpu The cross context virtual CPU structure.
5807 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5808 * out-of-sync. Make sure to update the required fields
5809 * before using them.
5810 */
5811DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5812{
5813 NOREF(pMixedCtx);
5814 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5815 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5816 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5817 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5818}
5819
5820
5821/**
5822 * Handle a condition that occurred while delivering an event through the guest
5823 * IDT.
5824 *
5825 * @returns Strict VBox status code (i.e. informational status codes too).
5826 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5827 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5828 * to continue execution of the guest which will delivery the \#DF.
5829 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5830 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5831 *
5832 * @param pVCpu The cross context virtual CPU structure.
5833 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5834 * out-of-sync. Make sure to update the required fields
5835 * before using them.
5836 * @param pVmxTransient Pointer to the VMX transient structure.
5837 *
5838 * @remarks No-long-jump zone!!!
5839 */
5840static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5841{
5842 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5843
5844 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5845 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5846
5847 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5848 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5849 {
5850 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5851 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5852
5853 typedef enum
5854 {
5855 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5856 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5857 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5858 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5859 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5860 } VMXREFLECTXCPT;
5861
5862 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5863 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5864 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5865 {
5866 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5867 {
5868 enmReflect = VMXREFLECTXCPT_XCPT;
5869#ifdef VBOX_STRICT
5870 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5871 && uExitVector == X86_XCPT_PF)
5872 {
5873 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5874 }
5875#endif
5876 if ( uExitVector == X86_XCPT_PF
5877 && uIdtVector == X86_XCPT_PF)
5878 {
5879 pVmxTransient->fVectoringDoublePF = true;
5880 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5881 }
5882 else if ( uExitVector == X86_XCPT_AC
5883 && uIdtVector == X86_XCPT_AC)
5884 {
5885 enmReflect = VMXREFLECTXCPT_HANG;
5886 Log4(("IDT: Nested #AC - Bad guest\n"));
5887 }
5888 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5889 && hmR0VmxIsContributoryXcpt(uExitVector)
5890 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5891 || uIdtVector == X86_XCPT_PF))
5892 {
5893 enmReflect = VMXREFLECTXCPT_DF;
5894 }
5895 else if (uIdtVector == X86_XCPT_DF)
5896 enmReflect = VMXREFLECTXCPT_TF;
5897 }
5898 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5899 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5900 {
5901 /*
5902 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5903 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5904 */
5905 enmReflect = VMXREFLECTXCPT_XCPT;
5906
5907 if (uExitVector == X86_XCPT_PF)
5908 {
5909 pVmxTransient->fVectoringPF = true;
5910 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5911 }
5912 }
5913 }
5914 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5915 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5916 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5917 {
5918 /*
5919 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5920 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5921 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5922 */
5923 enmReflect = VMXREFLECTXCPT_XCPT;
5924 }
5925
5926 /*
5927 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5928 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5929 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5930 *
5931 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5932 */
5933 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5934 && enmReflect == VMXREFLECTXCPT_XCPT
5935 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5936 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5937 {
5938 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5939 }
5940
5941 switch (enmReflect)
5942 {
5943 case VMXREFLECTXCPT_XCPT:
5944 {
5945 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5946 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5947 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5948
5949 uint32_t u32ErrCode = 0;
5950 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5951 {
5952 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5953 AssertRCReturn(rc2, rc2);
5954 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5955 }
5956
5957 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5958 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5959 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5960 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5961 rcStrict = VINF_SUCCESS;
5962 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5963 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5964
5965 break;
5966 }
5967
5968 case VMXREFLECTXCPT_DF:
5969 {
5970 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5971 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5972 rcStrict = VINF_HM_DOUBLE_FAULT;
5973 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5974 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5975
5976 break;
5977 }
5978
5979 case VMXREFLECTXCPT_TF:
5980 {
5981 rcStrict = VINF_EM_RESET;
5982 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5983 uExitVector));
5984 break;
5985 }
5986
5987 case VMXREFLECTXCPT_HANG:
5988 {
5989 rcStrict = VERR_EM_GUEST_CPU_HANG;
5990 break;
5991 }
5992
5993 default:
5994 Assert(rcStrict == VINF_SUCCESS);
5995 break;
5996 }
5997 }
5998 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5999 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6000 && uExitVector != X86_XCPT_DF
6001 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6002 {
6003 /*
6004 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6005 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6006 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6007 */
6008 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6009 {
6010 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6011 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6012 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6013 }
6014 }
6015
6016 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6017 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6018 return rcStrict;
6019}
6020
6021
6022/**
6023 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6024 *
6025 * @returns VBox status code.
6026 * @param pVCpu The cross context virtual CPU structure.
6027 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6028 * out-of-sync. Make sure to update the required fields
6029 * before using them.
6030 *
6031 * @remarks No-long-jump zone!!!
6032 */
6033static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6034{
6035 NOREF(pMixedCtx);
6036
6037 /*
6038 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6039 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6040 */
6041 VMMRZCallRing3Disable(pVCpu);
6042 HM_DISABLE_PREEMPT();
6043
6044 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6045 {
6046 uint32_t uVal = 0;
6047 uint32_t uShadow = 0;
6048 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6049 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6050 AssertRCReturn(rc, rc);
6051
6052 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6053 CPUMSetGuestCR0(pVCpu, uVal);
6054 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6055 }
6056
6057 HM_RESTORE_PREEMPT();
6058 VMMRZCallRing3Enable(pVCpu);
6059 return VINF_SUCCESS;
6060}
6061
6062
6063/**
6064 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6065 *
6066 * @returns VBox status code.
6067 * @param pVCpu The cross context virtual CPU structure.
6068 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6069 * out-of-sync. Make sure to update the required fields
6070 * before using them.
6071 *
6072 * @remarks No-long-jump zone!!!
6073 */
6074static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6075{
6076 NOREF(pMixedCtx);
6077
6078 int rc = VINF_SUCCESS;
6079 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6080 {
6081 uint32_t uVal = 0;
6082 uint32_t uShadow = 0;
6083 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6084 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6085 AssertRCReturn(rc, rc);
6086
6087 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6088 CPUMSetGuestCR4(pVCpu, uVal);
6089 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6090 }
6091 return rc;
6092}
6093
6094
6095/**
6096 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6097 *
6098 * @returns VBox status code.
6099 * @param pVCpu The cross context virtual CPU structure.
6100 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6101 * out-of-sync. Make sure to update the required fields
6102 * before using them.
6103 *
6104 * @remarks No-long-jump zone!!!
6105 */
6106static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6107{
6108 int rc = VINF_SUCCESS;
6109 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6110 {
6111 uint64_t u64Val = 0;
6112 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6113 AssertRCReturn(rc, rc);
6114
6115 pMixedCtx->rip = u64Val;
6116 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6117 }
6118 return rc;
6119}
6120
6121
6122/**
6123 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6124 *
6125 * @returns VBox status code.
6126 * @param pVCpu The cross context virtual CPU structure.
6127 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6128 * out-of-sync. Make sure to update the required fields
6129 * before using them.
6130 *
6131 * @remarks No-long-jump zone!!!
6132 */
6133static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6134{
6135 int rc = VINF_SUCCESS;
6136 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6137 {
6138 uint64_t u64Val = 0;
6139 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6140 AssertRCReturn(rc, rc);
6141
6142 pMixedCtx->rsp = u64Val;
6143 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6144 }
6145 return rc;
6146}
6147
6148
6149/**
6150 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6151 *
6152 * @returns VBox status code.
6153 * @param pVCpu The cross context virtual CPU structure.
6154 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6155 * out-of-sync. Make sure to update the required fields
6156 * before using them.
6157 *
6158 * @remarks No-long-jump zone!!!
6159 */
6160static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6161{
6162 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6163 {
6164 uint32_t uVal = 0;
6165 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6166 AssertRCReturn(rc, rc);
6167
6168 pMixedCtx->eflags.u32 = uVal;
6169 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6170 {
6171 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6172 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6173
6174 pMixedCtx->eflags.Bits.u1VM = 0;
6175 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6176 }
6177
6178 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6179 }
6180 return VINF_SUCCESS;
6181}
6182
6183
6184/**
6185 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6186 * guest-CPU context.
6187 */
6188DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6189{
6190 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6191 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6192 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6193 return rc;
6194}
6195
6196
6197/**
6198 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6199 * from the guest-state area in the VMCS.
6200 *
6201 * @param pVCpu The cross context virtual CPU structure.
6202 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6203 * out-of-sync. Make sure to update the required fields
6204 * before using them.
6205 *
6206 * @remarks No-long-jump zone!!!
6207 */
6208static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6209{
6210 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6211 {
6212 uint32_t uIntrState = 0;
6213 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6214 AssertRC(rc);
6215
6216 if (!uIntrState)
6217 {
6218 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6219 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6220
6221 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6222 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6223 }
6224 else
6225 {
6226 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6227 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6228 {
6229 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6230 AssertRC(rc);
6231 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6232 AssertRC(rc);
6233
6234 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6235 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6236 }
6237 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6238 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6239
6240 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6241 {
6242 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6243 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6244 }
6245 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6246 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6247 }
6248
6249 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6250 }
6251}
6252
6253
6254/**
6255 * Saves the guest's activity state.
6256 *
6257 * @returns VBox status code.
6258 * @param pVCpu The cross context virtual CPU structure.
6259 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6260 * out-of-sync. Make sure to update the required fields
6261 * before using them.
6262 *
6263 * @remarks No-long-jump zone!!!
6264 */
6265static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6266{
6267 NOREF(pMixedCtx);
6268 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6269 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6270 return VINF_SUCCESS;
6271}
6272
6273
6274/**
6275 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6276 * the current VMCS into the guest-CPU context.
6277 *
6278 * @returns VBox status code.
6279 * @param pVCpu The cross context virtual CPU structure.
6280 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6281 * out-of-sync. Make sure to update the required fields
6282 * before using them.
6283 *
6284 * @remarks No-long-jump zone!!!
6285 */
6286static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6287{
6288 int rc = VINF_SUCCESS;
6289 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6290 {
6291 uint32_t u32Val = 0;
6292 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6293 pMixedCtx->SysEnter.cs = u32Val;
6294 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6295 }
6296
6297 uint64_t u64Val = 0;
6298 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6299 {
6300 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6301 pMixedCtx->SysEnter.eip = u64Val;
6302 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6303 }
6304 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6305 {
6306 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6307 pMixedCtx->SysEnter.esp = u64Val;
6308 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6309 }
6310 return rc;
6311}
6312
6313
6314/**
6315 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6316 * the CPU back into the guest-CPU context.
6317 *
6318 * @returns VBox status code.
6319 * @param pVCpu The cross context virtual CPU structure.
6320 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6321 * out-of-sync. Make sure to update the required fields
6322 * before using them.
6323 *
6324 * @remarks No-long-jump zone!!!
6325 */
6326static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6327{
6328 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6329 VMMRZCallRing3Disable(pVCpu);
6330 HM_DISABLE_PREEMPT();
6331
6332 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6333 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6334 {
6335 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6336 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6337 }
6338
6339 HM_RESTORE_PREEMPT();
6340 VMMRZCallRing3Enable(pVCpu);
6341
6342 return VINF_SUCCESS;
6343}
6344
6345
6346/**
6347 * Saves the auto load/store'd guest MSRs from the current VMCS into
6348 * the guest-CPU context.
6349 *
6350 * @returns VBox status code.
6351 * @param pVCpu The cross context virtual CPU structure.
6352 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6353 * out-of-sync. Make sure to update the required fields
6354 * before using them.
6355 *
6356 * @remarks No-long-jump zone!!!
6357 */
6358static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6359{
6360 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6361 return VINF_SUCCESS;
6362
6363 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6364 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6365 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6366 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6367 {
6368 switch (pMsr->u32Msr)
6369 {
6370 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6371 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6372 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6373 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6374 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6375 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6376 break;
6377
6378 default:
6379 {
6380 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6381 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6382 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6383 }
6384 }
6385 }
6386
6387 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6388 return VINF_SUCCESS;
6389}
6390
6391
6392/**
6393 * Saves the guest control registers from the current VMCS into the guest-CPU
6394 * context.
6395 *
6396 * @returns VBox status code.
6397 * @param pVCpu The cross context virtual CPU structure.
6398 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6399 * out-of-sync. Make sure to update the required fields
6400 * before using them.
6401 *
6402 * @remarks No-long-jump zone!!!
6403 */
6404static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6405{
6406 /* Guest CR0. Guest FPU. */
6407 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6408 AssertRCReturn(rc, rc);
6409
6410 /* Guest CR4. */
6411 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6412 AssertRCReturn(rc, rc);
6413
6414 /* Guest CR2 - updated always during the world-switch or in #PF. */
6415 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6416 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6417 {
6418 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6419 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6420
6421 PVM pVM = pVCpu->CTX_SUFF(pVM);
6422 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6423 || ( pVM->hm.s.fNestedPaging
6424 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6425 {
6426 uint64_t u64Val = 0;
6427 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6428 if (pMixedCtx->cr3 != u64Val)
6429 {
6430 CPUMSetGuestCR3(pVCpu, u64Val);
6431 if (VMMRZCallRing3IsEnabled(pVCpu))
6432 {
6433 PGMUpdateCR3(pVCpu, u64Val);
6434 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6435 }
6436 else
6437 {
6438 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6439 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6440 }
6441 }
6442
6443 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6444 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6445 {
6446 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6447 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6448 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6449 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6450 AssertRCReturn(rc, rc);
6451
6452 if (VMMRZCallRing3IsEnabled(pVCpu))
6453 {
6454 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6455 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6456 }
6457 else
6458 {
6459 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6460 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6461 }
6462 }
6463 }
6464
6465 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6466 }
6467
6468 /*
6469 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6470 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6471 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6472 *
6473 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6474 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6475 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6476 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6477 *
6478 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6479 */
6480 if (VMMRZCallRing3IsEnabled(pVCpu))
6481 {
6482 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6483 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6484
6485 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6486 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6487
6488 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6489 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6490 }
6491
6492 return rc;
6493}
6494
6495
6496/**
6497 * Reads a guest segment register from the current VMCS into the guest-CPU
6498 * context.
6499 *
6500 * @returns VBox status code.
6501 * @param pVCpu The cross context virtual CPU structure.
6502 * @param idxSel Index of the selector in the VMCS.
6503 * @param idxLimit Index of the segment limit in the VMCS.
6504 * @param idxBase Index of the segment base in the VMCS.
6505 * @param idxAccess Index of the access rights of the segment in the VMCS.
6506 * @param pSelReg Pointer to the segment selector.
6507 *
6508 * @remarks No-long-jump zone!!!
6509 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6510 * macro as that takes care of whether to read from the VMCS cache or
6511 * not.
6512 */
6513DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6514 PCPUMSELREG pSelReg)
6515{
6516 NOREF(pVCpu);
6517
6518 uint32_t u32Val = 0;
6519 int rc = VMXReadVmcs32(idxSel, &u32Val);
6520 AssertRCReturn(rc, rc);
6521 pSelReg->Sel = (uint16_t)u32Val;
6522 pSelReg->ValidSel = (uint16_t)u32Val;
6523 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6524
6525 rc = VMXReadVmcs32(idxLimit, &u32Val);
6526 AssertRCReturn(rc, rc);
6527 pSelReg->u32Limit = u32Val;
6528
6529 uint64_t u64Val = 0;
6530 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6531 AssertRCReturn(rc, rc);
6532 pSelReg->u64Base = u64Val;
6533
6534 rc = VMXReadVmcs32(idxAccess, &u32Val);
6535 AssertRCReturn(rc, rc);
6536 pSelReg->Attr.u = u32Val;
6537
6538 /*
6539 * If VT-x marks the segment as unusable, most other bits remain undefined:
6540 * - For CS the L, D and G bits have meaning.
6541 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6542 * - For the remaining data segments no bits are defined.
6543 *
6544 * The present bit and the unusable bit has been observed to be set at the
6545 * same time (the selector was supposed to be invalid as we started executing
6546 * a V8086 interrupt in ring-0).
6547 *
6548 * What should be important for the rest of the VBox code, is that the P bit is
6549 * cleared. Some of the other VBox code recognizes the unusable bit, but
6550 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6551 * safe side here, we'll strip off P and other bits we don't care about. If
6552 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6553 *
6554 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6555 */
6556 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6557 {
6558 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6559
6560 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6561 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6562 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6563
6564 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6565#ifdef DEBUG_bird
6566 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6567 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6568 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6569#endif
6570 }
6571 return VINF_SUCCESS;
6572}
6573
6574
6575#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6576# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6577 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6578 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6579#else
6580# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6581 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6582 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6583#endif
6584
6585
6586/**
6587 * Saves the guest segment registers from the current VMCS into the guest-CPU
6588 * context.
6589 *
6590 * @returns VBox status code.
6591 * @param pVCpu The cross context virtual CPU structure.
6592 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6593 * out-of-sync. Make sure to update the required fields
6594 * before using them.
6595 *
6596 * @remarks No-long-jump zone!!!
6597 */
6598static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6599{
6600 /* Guest segment registers. */
6601 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6602 {
6603 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6604 AssertRCReturn(rc, rc);
6605
6606 rc = VMXLOCAL_READ_SEG(CS, cs);
6607 rc |= VMXLOCAL_READ_SEG(SS, ss);
6608 rc |= VMXLOCAL_READ_SEG(DS, ds);
6609 rc |= VMXLOCAL_READ_SEG(ES, es);
6610 rc |= VMXLOCAL_READ_SEG(FS, fs);
6611 rc |= VMXLOCAL_READ_SEG(GS, gs);
6612 AssertRCReturn(rc, rc);
6613
6614 /* Restore segment attributes for real-on-v86 mode hack. */
6615 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6616 {
6617 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6618 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6619 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6620 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6621 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6622 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6623 }
6624 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6625 }
6626
6627 return VINF_SUCCESS;
6628}
6629
6630
6631/**
6632 * Saves the guest descriptor table registers and task register from the current
6633 * VMCS into the guest-CPU context.
6634 *
6635 * @returns VBox status code.
6636 * @param pVCpu The cross context virtual CPU structure.
6637 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6638 * out-of-sync. Make sure to update the required fields
6639 * before using them.
6640 *
6641 * @remarks No-long-jump zone!!!
6642 */
6643static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6644{
6645 int rc = VINF_SUCCESS;
6646
6647 /* Guest LDTR. */
6648 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6649 {
6650 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6651 AssertRCReturn(rc, rc);
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6653 }
6654
6655 /* Guest GDTR. */
6656 uint64_t u64Val = 0;
6657 uint32_t u32Val = 0;
6658 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6659 {
6660 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6661 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6662 pMixedCtx->gdtr.pGdt = u64Val;
6663 pMixedCtx->gdtr.cbGdt = u32Val;
6664 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6665 }
6666
6667 /* Guest IDTR. */
6668 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6669 {
6670 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6671 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6672 pMixedCtx->idtr.pIdt = u64Val;
6673 pMixedCtx->idtr.cbIdt = u32Val;
6674 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6675 }
6676
6677 /* Guest TR. */
6678 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6679 {
6680 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6681 AssertRCReturn(rc, rc);
6682
6683 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6684 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6685 {
6686 rc = VMXLOCAL_READ_SEG(TR, tr);
6687 AssertRCReturn(rc, rc);
6688 }
6689 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6690 }
6691 return rc;
6692}
6693
6694#undef VMXLOCAL_READ_SEG
6695
6696
6697/**
6698 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6699 * context.
6700 *
6701 * @returns VBox status code.
6702 * @param pVCpu The cross context virtual CPU structure.
6703 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6704 * out-of-sync. Make sure to update the required fields
6705 * before using them.
6706 *
6707 * @remarks No-long-jump zone!!!
6708 */
6709static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6710{
6711 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6712 {
6713 if (!pVCpu->hm.s.fUsingHyperDR7)
6714 {
6715 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6716 uint32_t u32Val;
6717 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6718 pMixedCtx->dr[7] = u32Val;
6719 }
6720
6721 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6722 }
6723 return VINF_SUCCESS;
6724}
6725
6726
6727/**
6728 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6729 *
6730 * @returns VBox status code.
6731 * @param pVCpu The cross context virtual CPU structure.
6732 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6733 * out-of-sync. Make sure to update the required fields
6734 * before using them.
6735 *
6736 * @remarks No-long-jump zone!!!
6737 */
6738static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6739{
6740 NOREF(pMixedCtx);
6741
6742 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6743 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6744 return VINF_SUCCESS;
6745}
6746
6747
6748/**
6749 * Saves the entire guest state from the currently active VMCS into the
6750 * guest-CPU context.
6751 *
6752 * This essentially VMREADs all guest-data.
6753 *
6754 * @returns VBox status code.
6755 * @param pVCpu The cross context virtual CPU structure.
6756 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6757 * out-of-sync. Make sure to update the required fields
6758 * before using them.
6759 */
6760static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6761{
6762 Assert(pVCpu);
6763 Assert(pMixedCtx);
6764
6765 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6766 return VINF_SUCCESS;
6767
6768 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6769 again on the ring-3 callback path, there is no real need to. */
6770 if (VMMRZCallRing3IsEnabled(pVCpu))
6771 VMMR0LogFlushDisable(pVCpu);
6772 else
6773 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6774 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6775
6776 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6777 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6778
6779 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6780 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6781
6782 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6783 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6784
6785 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6786 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6787
6788 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6789 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6790
6791 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6792 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6793
6794 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6795 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6796
6797 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6798 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6799
6800 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6801 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6802
6803 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6804 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6805
6806 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6807 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6808 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6809
6810 if (VMMRZCallRing3IsEnabled(pVCpu))
6811 VMMR0LogFlushEnable(pVCpu);
6812
6813 return VINF_SUCCESS;
6814}
6815
6816
6817/**
6818 * Saves basic guest registers needed for IEM instruction execution.
6819 *
6820 * @returns VBox status code (OR-able).
6821 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6822 * @param pMixedCtx Pointer to the CPU context of the guest.
6823 * @param fMemory Whether the instruction being executed operates on
6824 * memory or not. Only CR0 is synced up if clear.
6825 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6826 */
6827static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6828{
6829 /*
6830 * We assume all general purpose registers other than RSP are available.
6831 *
6832 * - RIP is a must, as it will be incremented or otherwise changed.
6833 * - RFLAGS are always required to figure the CPL.
6834 * - RSP isn't always required, however it's a GPR, so frequently required.
6835 * - SS and CS are the only segment register needed if IEM doesn't do memory
6836 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6837 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6838 * be required for memory accesses.
6839 *
6840 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6841 */
6842 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6843 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6844 if (fNeedRsp)
6845 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6846 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6847 if (!fMemory)
6848 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6849 else
6850 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6851 AssertRCReturn(rc, rc);
6852 return rc;
6853}
6854
6855
6856/**
6857 * Ensures that we've got a complete basic guest-context.
6858 *
6859 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6860 * is for the interpreter.
6861 *
6862 * @returns VBox status code.
6863 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6864 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6865 * needing to be synced in.
6866 * @thread EMT(pVCpu)
6867 */
6868VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6869{
6870 /* Note! Since this is only applicable to VT-x, the implementation is placed
6871 in the VT-x part of the sources instead of the generic stuff. */
6872 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6873 {
6874 /* For now, imply that the caller might change everything too. */
6875 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6876 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6877 }
6878 return VINF_SUCCESS;
6879}
6880
6881
6882/**
6883 * Check per-VM and per-VCPU force flag actions that require us to go back to
6884 * ring-3 for one reason or another.
6885 *
6886 * @returns Strict VBox status code (i.e. informational status codes too)
6887 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6888 * ring-3.
6889 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6890 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6891 * interrupts)
6892 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6893 * all EMTs to be in ring-3.
6894 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6895 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6896 * to the EM loop.
6897 *
6898 * @param pVM The cross context VM structure.
6899 * @param pVCpu The cross context virtual CPU structure.
6900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6901 * out-of-sync. Make sure to update the required fields
6902 * before using them.
6903 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6904 */
6905static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6906{
6907 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6908
6909 /*
6910 * Anything pending? Should be more likely than not if we're doing a good job.
6911 */
6912 if ( !fStepping
6913 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6914 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6915 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6916 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6917 return VINF_SUCCESS;
6918
6919 /* We need the control registers now, make sure the guest-CPU context is updated. */
6920 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6921 AssertRCReturn(rc3, rc3);
6922
6923 /* Pending HM CR3 sync. */
6924 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6925 {
6926 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6927 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6928 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6929 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6930 }
6931
6932 /* Pending HM PAE PDPEs. */
6933 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6934 {
6935 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6936 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6937 }
6938
6939 /* Pending PGM C3 sync. */
6940 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6941 {
6942 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6943 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6944 if (rcStrict2 != VINF_SUCCESS)
6945 {
6946 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6947 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6948 return rcStrict2;
6949 }
6950 }
6951
6952 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6953 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6954 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6955 {
6956 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6957 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6958 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6959 return rc2;
6960 }
6961
6962 /* Pending VM request packets, such as hardware interrupts. */
6963 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6964 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6965 {
6966 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6967 return VINF_EM_PENDING_REQUEST;
6968 }
6969
6970 /* Pending PGM pool flushes. */
6971 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6972 {
6973 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6974 return VINF_PGM_POOL_FLUSH_PENDING;
6975 }
6976
6977 /* Pending DMA requests. */
6978 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6979 {
6980 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6981 return VINF_EM_RAW_TO_R3;
6982 }
6983
6984 return VINF_SUCCESS;
6985}
6986
6987
6988/**
6989 * Converts any TRPM trap into a pending HM event. This is typically used when
6990 * entering from ring-3 (not longjmp returns).
6991 *
6992 * @param pVCpu The cross context virtual CPU structure.
6993 */
6994static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6995{
6996 Assert(TRPMHasTrap(pVCpu));
6997 Assert(!pVCpu->hm.s.Event.fPending);
6998
6999 uint8_t uVector;
7000 TRPMEVENT enmTrpmEvent;
7001 RTGCUINT uErrCode;
7002 RTGCUINTPTR GCPtrFaultAddress;
7003 uint8_t cbInstr;
7004
7005 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7006 AssertRC(rc);
7007
7008 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7009 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7010 if (enmTrpmEvent == TRPM_TRAP)
7011 {
7012 switch (uVector)
7013 {
7014 case X86_XCPT_NMI:
7015 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7016 break;
7017
7018 case X86_XCPT_BP:
7019 case X86_XCPT_OF:
7020 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7021 break;
7022
7023 case X86_XCPT_PF:
7024 case X86_XCPT_DF:
7025 case X86_XCPT_TS:
7026 case X86_XCPT_NP:
7027 case X86_XCPT_SS:
7028 case X86_XCPT_GP:
7029 case X86_XCPT_AC:
7030 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7031 /* no break! */
7032 default:
7033 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7034 break;
7035 }
7036 }
7037 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7038 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7039 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7040 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7041 else
7042 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7043
7044 rc = TRPMResetTrap(pVCpu);
7045 AssertRC(rc);
7046 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7047 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7048
7049 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7050}
7051
7052
7053/**
7054 * Converts the pending HM event into a TRPM trap.
7055 *
7056 * @param pVCpu The cross context virtual CPU structure.
7057 */
7058static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7059{
7060 Assert(pVCpu->hm.s.Event.fPending);
7061
7062 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7063 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7064 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7065 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7066
7067 /* If a trap was already pending, we did something wrong! */
7068 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7069
7070 TRPMEVENT enmTrapType;
7071 switch (uVectorType)
7072 {
7073 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7074 enmTrapType = TRPM_HARDWARE_INT;
7075 break;
7076
7077 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7078 enmTrapType = TRPM_SOFTWARE_INT;
7079 break;
7080
7081 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7082 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7083 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7084 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7085 enmTrapType = TRPM_TRAP;
7086 break;
7087
7088 default:
7089 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7090 enmTrapType = TRPM_32BIT_HACK;
7091 break;
7092 }
7093
7094 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7095
7096 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7097 AssertRC(rc);
7098
7099 if (fErrorCodeValid)
7100 TRPMSetErrorCode(pVCpu, uErrorCode);
7101
7102 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7103 && uVector == X86_XCPT_PF)
7104 {
7105 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7106 }
7107 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7108 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7109 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7110 {
7111 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7112 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7113 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7114 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7115 }
7116
7117 /* Clear any pending events from the VMCS. */
7118 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7119 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7120
7121 /* We're now done converting the pending event. */
7122 pVCpu->hm.s.Event.fPending = false;
7123}
7124
7125
7126/**
7127 * Does the necessary state syncing before returning to ring-3 for any reason
7128 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7129 *
7130 * @returns VBox status code.
7131 * @param pVCpu The cross context virtual CPU structure.
7132 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7133 * be out-of-sync. Make sure to update the required
7134 * fields before using them.
7135 * @param fSaveGuestState Whether to save the guest state or not.
7136 *
7137 * @remarks No-long-jmp zone!!!
7138 */
7139static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7140{
7141 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7142 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7143
7144 RTCPUID idCpu = RTMpCpuId();
7145 Log4Func(("HostCpuId=%u\n", idCpu));
7146
7147 /*
7148 * !!! IMPORTANT !!!
7149 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7150 */
7151
7152 /* Save the guest state if necessary. */
7153 if ( fSaveGuestState
7154 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7155 {
7156 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7157 AssertRCReturn(rc, rc);
7158 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7159 }
7160
7161 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7162 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7163 {
7164 if (fSaveGuestState)
7165 {
7166 /* We shouldn't reload CR0 without saving it first. */
7167 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7168 AssertRCReturn(rc, rc);
7169 }
7170 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7171 }
7172
7173 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7174#ifdef VBOX_STRICT
7175 if (CPUMIsHyperDebugStateActive(pVCpu))
7176 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7177#endif
7178 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7179 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7180 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7181 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7182
7183#if HC_ARCH_BITS == 64
7184 /* Restore host-state bits that VT-x only restores partially. */
7185 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7186 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7187 {
7188 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7189 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7190 }
7191 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7192#endif
7193
7194 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7195 if (pVCpu->hm.s.vmx.fLazyMsrs)
7196 {
7197 /* We shouldn't reload the guest MSRs without saving it first. */
7198 if (!fSaveGuestState)
7199 {
7200 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7201 AssertRCReturn(rc, rc);
7202 }
7203 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7204 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7205 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7206 }
7207
7208 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7209 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7210
7211 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7212 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7213 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7214 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7215 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7216 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7218 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7219
7220 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7221
7222 /** @todo This partially defeats the purpose of having preemption hooks.
7223 * The problem is, deregistering the hooks should be moved to a place that
7224 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7225 * context.
7226 */
7227 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7228 {
7229 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7230 AssertRCReturn(rc, rc);
7231
7232 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7233 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7234 }
7235 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7236 NOREF(idCpu);
7237
7238 return VINF_SUCCESS;
7239}
7240
7241
7242/**
7243 * Leaves the VT-x session.
7244 *
7245 * @returns VBox status code.
7246 * @param pVCpu The cross context virtual CPU structure.
7247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7248 * out-of-sync. Make sure to update the required fields
7249 * before using them.
7250 *
7251 * @remarks No-long-jmp zone!!!
7252 */
7253DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7254{
7255 HM_DISABLE_PREEMPT();
7256 HMVMX_ASSERT_CPU_SAFE();
7257 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7258 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7259
7260 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7261 and done this from the VMXR0ThreadCtxCallback(). */
7262 if (!pVCpu->hm.s.fLeaveDone)
7263 {
7264 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7265 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7266 pVCpu->hm.s.fLeaveDone = true;
7267 }
7268 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7269
7270 /*
7271 * !!! IMPORTANT !!!
7272 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7273 */
7274
7275 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7276 /** @todo Deregistering here means we need to VMCLEAR always
7277 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7278 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7279 VMMR0ThreadCtxHookDisable(pVCpu);
7280
7281 /* Leave HM context. This takes care of local init (term). */
7282 int rc = HMR0LeaveCpu(pVCpu);
7283
7284 HM_RESTORE_PREEMPT();
7285 return rc;
7286}
7287
7288
7289/**
7290 * Does the necessary state syncing before doing a longjmp to ring-3.
7291 *
7292 * @returns VBox status code.
7293 * @param pVCpu The cross context virtual CPU structure.
7294 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7295 * out-of-sync. Make sure to update the required fields
7296 * before using them.
7297 *
7298 * @remarks No-long-jmp zone!!!
7299 */
7300DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7301{
7302 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7303}
7304
7305
7306/**
7307 * Take necessary actions before going back to ring-3.
7308 *
7309 * An action requires us to go back to ring-3. This function does the necessary
7310 * steps before we can safely return to ring-3. This is not the same as longjmps
7311 * to ring-3, this is voluntary and prepares the guest so it may continue
7312 * executing outside HM (recompiler/IEM).
7313 *
7314 * @returns VBox status code.
7315 * @param pVM The cross context VM structure.
7316 * @param pVCpu The cross context virtual CPU structure.
7317 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7318 * out-of-sync. Make sure to update the required fields
7319 * before using them.
7320 * @param rcExit The reason for exiting to ring-3. Can be
7321 * VINF_VMM_UNKNOWN_RING3_CALL.
7322 */
7323static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7324{
7325 Assert(pVM);
7326 Assert(pVCpu);
7327 Assert(pMixedCtx);
7328 HMVMX_ASSERT_PREEMPT_SAFE();
7329
7330 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7331 {
7332 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7333 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7334 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7335 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7336 }
7337
7338 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7339 VMMRZCallRing3Disable(pVCpu);
7340 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7341
7342 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7343 if (pVCpu->hm.s.Event.fPending)
7344 {
7345 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7346 Assert(!pVCpu->hm.s.Event.fPending);
7347 }
7348
7349 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7350 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7351
7352 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7353 and if we're injecting an event we should have a TRPM trap pending. */
7354 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7355#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7356 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7357#endif
7358
7359 /* Save guest state and restore host state bits. */
7360 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7361 AssertRCReturn(rc, rc);
7362 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7363 /* Thread-context hooks are unregistered at this point!!! */
7364
7365 /* Sync recompiler state. */
7366 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7367 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7368 | CPUM_CHANGED_LDTR
7369 | CPUM_CHANGED_GDTR
7370 | CPUM_CHANGED_IDTR
7371 | CPUM_CHANGED_TR
7372 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7373 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7374 if ( pVM->hm.s.fNestedPaging
7375 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7376 {
7377 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7378 }
7379
7380 Assert(!pVCpu->hm.s.fClearTrapFlag);
7381
7382 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7383 if (rcExit != VINF_EM_RAW_INTERRUPT)
7384 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7385
7386 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7387
7388 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7389 VMMRZCallRing3RemoveNotification(pVCpu);
7390 VMMRZCallRing3Enable(pVCpu);
7391
7392 return rc;
7393}
7394
7395
7396/**
7397 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7398 * longjump to ring-3 and possibly get preempted.
7399 *
7400 * @returns VBox status code.
7401 * @param pVCpu The cross context virtual CPU structure.
7402 * @param enmOperation The operation causing the ring-3 longjump.
7403 * @param pvUser Opaque pointer to the guest-CPU context. The data
7404 * may be out-of-sync. Make sure to update the required
7405 * fields before using them.
7406 */
7407static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7408{
7409 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7410 {
7411 /*
7412 * !!! IMPORTANT !!!
7413 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7414 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7415 */
7416 VMMRZCallRing3RemoveNotification(pVCpu);
7417 VMMRZCallRing3Disable(pVCpu);
7418 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7419 RTThreadPreemptDisable(&PreemptState);
7420
7421 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7422 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7423
7424#if HC_ARCH_BITS == 64
7425 /* Restore host-state bits that VT-x only restores partially. */
7426 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7427 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7428 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7429 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7430#endif
7431 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7432 if (pVCpu->hm.s.vmx.fLazyMsrs)
7433 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7434
7435 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7436 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7437 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7438 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7439 {
7440 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7441 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7442 }
7443
7444 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7445 VMMR0ThreadCtxHookDisable(pVCpu);
7446 HMR0LeaveCpu(pVCpu);
7447 RTThreadPreemptRestore(&PreemptState);
7448 return VINF_SUCCESS;
7449 }
7450
7451 Assert(pVCpu);
7452 Assert(pvUser);
7453 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7454 HMVMX_ASSERT_PREEMPT_SAFE();
7455
7456 VMMRZCallRing3Disable(pVCpu);
7457 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7458
7459 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7460 enmOperation));
7461
7462 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7463 AssertRCReturn(rc, rc);
7464
7465 VMMRZCallRing3Enable(pVCpu);
7466 return VINF_SUCCESS;
7467}
7468
7469
7470/**
7471 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7472 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7473 *
7474 * @param pVCpu The cross context virtual CPU structure.
7475 */
7476DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7477{
7478 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7479 {
7480 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7481 {
7482 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7484 AssertRC(rc);
7485 Log4(("Setup interrupt-window exiting\n"));
7486 }
7487 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7488}
7489
7490
7491/**
7492 * Clears the interrupt-window exiting control in the VMCS.
7493 *
7494 * @param pVCpu The cross context virtual CPU structure.
7495 */
7496DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7497{
7498 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7499 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7500 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7501 AssertRC(rc);
7502 Log4(("Cleared interrupt-window exiting\n"));
7503}
7504
7505
7506/**
7507 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7508 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7509 *
7510 * @param pVCpu The cross context virtual CPU structure.
7511 */
7512DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7513{
7514 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7515 {
7516 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7517 {
7518 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7519 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7520 AssertRC(rc);
7521 Log4(("Setup NMI-window exiting\n"));
7522 }
7523 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7524}
7525
7526
7527/**
7528 * Clears the NMI-window exiting control in the VMCS.
7529 *
7530 * @param pVCpu The cross context virtual CPU structure.
7531 */
7532DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7533{
7534 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7535 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7536 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7537 AssertRC(rc);
7538 Log4(("Cleared NMI-window exiting\n"));
7539}
7540
7541
7542/**
7543 * Evaluates the event to be delivered to the guest and sets it as the pending
7544 * event.
7545 *
7546 * @returns The VT-x guest-interruptibility state.
7547 * @param pVCpu The cross context virtual CPU structure.
7548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7549 * out-of-sync. Make sure to update the required fields
7550 * before using them.
7551 */
7552static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7553{
7554 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7555 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7556 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7557 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7558 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7559
7560 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7561 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7562 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7563 Assert(!TRPMHasTrap(pVCpu));
7564
7565 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7566 APICUpdatePendingInterrupts(pVCpu);
7567
7568 /*
7569 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7570 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7571 */
7572 /** @todo SMI. SMIs take priority over NMIs. */
7573 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7574 {
7575 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7576 if ( !pVCpu->hm.s.Event.fPending
7577 && !fBlockNmi
7578 && !fBlockSti
7579 && !fBlockMovSS)
7580 {
7581 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7582 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7583 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7584
7585 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7586 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7587 }
7588 else
7589 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7590 }
7591 /*
7592 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7593 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7594 */
7595 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7596 && !pVCpu->hm.s.fSingleInstruction)
7597 {
7598 Assert(!DBGFIsStepping(pVCpu));
7599 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7600 AssertRC(rc);
7601 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7602 if ( !pVCpu->hm.s.Event.fPending
7603 && !fBlockInt
7604 && !fBlockSti
7605 && !fBlockMovSS)
7606 {
7607 uint8_t u8Interrupt;
7608 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7609 if (RT_SUCCESS(rc))
7610 {
7611 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7612 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7613 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7614
7615 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7616 }
7617 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7618 {
7619 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7620 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7621 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7622
7623 /*
7624 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7625 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7626 * need to re-set this force-flag here.
7627 */
7628 }
7629 else
7630 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7631 }
7632 else
7633 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7634 }
7635
7636 return uIntrState;
7637}
7638
7639
7640/**
7641 * Sets a pending-debug exception to be delivered to the guest if the guest is
7642 * single-stepping in the VMCS.
7643 *
7644 * @param pVCpu The cross context virtual CPU structure.
7645 */
7646DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7647{
7648 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7649 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7650 AssertRC(rc);
7651}
7652
7653
7654/**
7655 * Injects any pending events into the guest if the guest is in a state to
7656 * receive them.
7657 *
7658 * @returns Strict VBox status code (i.e. informational status codes too).
7659 * @param pVCpu The cross context virtual CPU structure.
7660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7661 * out-of-sync. Make sure to update the required fields
7662 * before using them.
7663 * @param uIntrState The VT-x guest-interruptibility state.
7664 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7665 * return VINF_EM_DBG_STEPPED if the event was
7666 * dispatched directly.
7667 */
7668static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7669{
7670 HMVMX_ASSERT_PREEMPT_SAFE();
7671 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7672
7673 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7674 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7675
7676 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7677 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7678 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7679 Assert(!TRPMHasTrap(pVCpu));
7680
7681 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7682 if (pVCpu->hm.s.Event.fPending)
7683 {
7684 /*
7685 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7686 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7687 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7688 *
7689 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7690 */
7691 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7692#ifdef VBOX_STRICT
7693 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7694 {
7695 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7696 Assert(!fBlockInt);
7697 Assert(!fBlockSti);
7698 Assert(!fBlockMovSS);
7699 }
7700 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7701 {
7702 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7703 Assert(!fBlockSti);
7704 Assert(!fBlockMovSS);
7705 Assert(!fBlockNmi);
7706 }
7707#endif
7708 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7709 (uint8_t)uIntType));
7710 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7711 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7712 fStepping, &uIntrState);
7713 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7714
7715 /* Update the interruptibility-state as it could have been changed by
7716 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7717 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7718 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7719
7720 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7721 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7722 else
7723 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7724 }
7725
7726 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7727 if ( fBlockSti
7728 || fBlockMovSS)
7729 {
7730 if (!pVCpu->hm.s.fSingleInstruction)
7731 {
7732 /*
7733 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7734 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7735 * See Intel spec. 27.3.4 "Saving Non-Register State".
7736 */
7737 Assert(!DBGFIsStepping(pVCpu));
7738 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7739 AssertRCReturn(rc2, rc2);
7740 if (pMixedCtx->eflags.Bits.u1TF)
7741 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7742 }
7743 else if (pMixedCtx->eflags.Bits.u1TF)
7744 {
7745 /*
7746 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7747 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7748 */
7749 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7750 uIntrState = 0;
7751 }
7752 }
7753
7754 /*
7755 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7756 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7757 */
7758 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7759 AssertRC(rc2);
7760
7761 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7762 NOREF(fBlockMovSS); NOREF(fBlockSti);
7763 return rcStrict;
7764}
7765
7766
7767/**
7768 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7769 *
7770 * @param pVCpu The cross context virtual CPU structure.
7771 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7772 * out-of-sync. Make sure to update the required fields
7773 * before using them.
7774 */
7775DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7776{
7777 NOREF(pMixedCtx);
7778 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7779 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7780}
7781
7782
7783/**
7784 * Injects a double-fault (\#DF) exception into the VM.
7785 *
7786 * @returns Strict VBox status code (i.e. informational status codes too).
7787 * @param pVCpu The cross context virtual CPU structure.
7788 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7789 * out-of-sync. Make sure to update the required fields
7790 * before using them.
7791 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7792 * and should return VINF_EM_DBG_STEPPED if the event
7793 * is injected directly (register modified by us, not
7794 * by hardware on VM-entry).
7795 * @param puIntrState Pointer to the current guest interruptibility-state.
7796 * This interruptibility-state will be updated if
7797 * necessary. This cannot not be NULL.
7798 */
7799DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7800{
7801 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7802 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7803 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7804 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7805 fStepping, puIntrState);
7806}
7807
7808
7809/**
7810 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7811 *
7812 * @param pVCpu The cross context virtual CPU structure.
7813 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7814 * out-of-sync. Make sure to update the required fields
7815 * before using them.
7816 */
7817DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7818{
7819 NOREF(pMixedCtx);
7820 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7821 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7822 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7823}
7824
7825
7826/**
7827 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7828 *
7829 * @param pVCpu The cross context virtual CPU structure.
7830 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7831 * out-of-sync. Make sure to update the required fields
7832 * before using them.
7833 * @param cbInstr The value of RIP that is to be pushed on the guest
7834 * stack.
7835 */
7836DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7837{
7838 NOREF(pMixedCtx);
7839 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7840 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7841 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7842}
7843
7844
7845/**
7846 * Injects a general-protection (\#GP) fault into the VM.
7847 *
7848 * @returns Strict VBox status code (i.e. informational status codes too).
7849 * @param pVCpu The cross context virtual CPU structure.
7850 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7851 * out-of-sync. Make sure to update the required fields
7852 * before using them.
7853 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7854 * mode, i.e. in real-mode it's not valid).
7855 * @param u32ErrorCode The error code associated with the \#GP.
7856 * @param fStepping Whether we're running in
7857 * hmR0VmxRunGuestCodeStep() and should return
7858 * VINF_EM_DBG_STEPPED if the event is injected
7859 * directly (register modified by us, not by
7860 * hardware on VM-entry).
7861 * @param puIntrState Pointer to the current guest interruptibility-state.
7862 * This interruptibility-state will be updated if
7863 * necessary. This cannot not be NULL.
7864 */
7865DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7866 bool fStepping, uint32_t *puIntrState)
7867{
7868 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7869 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7870 if (fErrorCodeValid)
7871 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7872 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7873 fStepping, puIntrState);
7874}
7875
7876
7877#if 0 /* unused */
7878/**
7879 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7880 * VM.
7881 *
7882 * @param pVCpu The cross context virtual CPU structure.
7883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7884 * out-of-sync. Make sure to update the required fields
7885 * before using them.
7886 * @param u32ErrorCode The error code associated with the \#GP.
7887 */
7888DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7889{
7890 NOREF(pMixedCtx);
7891 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7892 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7893 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7894 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7895}
7896#endif /* unused */
7897
7898
7899/**
7900 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7901 *
7902 * @param pVCpu The cross context virtual CPU structure.
7903 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7904 * out-of-sync. Make sure to update the required fields
7905 * before using them.
7906 * @param uVector The software interrupt vector number.
7907 * @param cbInstr The value of RIP that is to be pushed on the guest
7908 * stack.
7909 */
7910DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7911{
7912 NOREF(pMixedCtx);
7913 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7914 if ( uVector == X86_XCPT_BP
7915 || uVector == X86_XCPT_OF)
7916 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7917 else
7918 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7919 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7920}
7921
7922
7923/**
7924 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7925 * stack.
7926 *
7927 * @returns Strict VBox status code (i.e. informational status codes too).
7928 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7929 * @param pVM The cross context VM structure.
7930 * @param pMixedCtx Pointer to the guest-CPU context.
7931 * @param uValue The value to push to the guest stack.
7932 */
7933DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7934{
7935 /*
7936 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7937 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7938 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7939 */
7940 if (pMixedCtx->sp == 1)
7941 return VINF_EM_RESET;
7942 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7943 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7944 AssertRC(rc);
7945 return rc;
7946}
7947
7948
7949/**
7950 * Injects an event into the guest upon VM-entry by updating the relevant fields
7951 * in the VM-entry area in the VMCS.
7952 *
7953 * @returns Strict VBox status code (i.e. informational status codes too).
7954 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7955 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7956 *
7957 * @param pVCpu The cross context virtual CPU structure.
7958 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7959 * be out-of-sync. Make sure to update the required
7960 * fields before using them.
7961 * @param u64IntInfo The VM-entry interruption-information field.
7962 * @param cbInstr The VM-entry instruction length in bytes (for
7963 * software interrupts, exceptions and privileged
7964 * software exceptions).
7965 * @param u32ErrCode The VM-entry exception error code.
7966 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7967 * @param puIntrState Pointer to the current guest interruptibility-state.
7968 * This interruptibility-state will be updated if
7969 * necessary. This cannot not be NULL.
7970 * @param fStepping Whether we're running in
7971 * hmR0VmxRunGuestCodeStep() and should return
7972 * VINF_EM_DBG_STEPPED if the event is injected
7973 * directly (register modified by us, not by
7974 * hardware on VM-entry).
7975 *
7976 * @remarks Requires CR0!
7977 * @remarks No-long-jump zone!!!
7978 */
7979static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7980 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7981 uint32_t *puIntrState)
7982{
7983 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7984 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7985 Assert(puIntrState);
7986 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7987
7988 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7989 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7990
7991#ifdef VBOX_STRICT
7992 /* Validate the error-code-valid bit for hardware exceptions. */
7993 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7994 {
7995 switch (uVector)
7996 {
7997 case X86_XCPT_PF:
7998 case X86_XCPT_DF:
7999 case X86_XCPT_TS:
8000 case X86_XCPT_NP:
8001 case X86_XCPT_SS:
8002 case X86_XCPT_GP:
8003 case X86_XCPT_AC:
8004 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8005 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8006 /* fallthru */
8007 default:
8008 break;
8009 }
8010 }
8011#endif
8012
8013 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8014 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8015 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8016
8017 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8018
8019 /* We require CR0 to check if the guest is in real-mode. */
8020 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8021 AssertRCReturn(rc, rc);
8022
8023 /*
8024 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8025 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8026 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8027 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8028 */
8029 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8030 {
8031 PVM pVM = pVCpu->CTX_SUFF(pVM);
8032 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8033 {
8034 Assert(PDMVmmDevHeapIsEnabled(pVM));
8035 Assert(pVM->hm.s.vmx.pRealModeTSS);
8036
8037 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8038 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8039 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8040 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8041 AssertRCReturn(rc, rc);
8042 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8043
8044 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8045 size_t const cbIdtEntry = sizeof(X86IDTR16);
8046 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8047 {
8048 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8049 if (uVector == X86_XCPT_DF)
8050 return VINF_EM_RESET;
8051
8052 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8053 if (uVector == X86_XCPT_GP)
8054 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8055
8056 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8057 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8058 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8059 fStepping, puIntrState);
8060 }
8061
8062 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8063 uint16_t uGuestIp = pMixedCtx->ip;
8064 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8065 {
8066 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8067 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8068 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8069 }
8070 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8071 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8072
8073 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8074 X86IDTR16 IdtEntry;
8075 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8076 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8077 AssertRCReturn(rc, rc);
8078
8079 /* Construct the stack frame for the interrupt/exception handler. */
8080 VBOXSTRICTRC rcStrict;
8081 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8082 if (rcStrict == VINF_SUCCESS)
8083 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8084 if (rcStrict == VINF_SUCCESS)
8085 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8086
8087 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8088 if (rcStrict == VINF_SUCCESS)
8089 {
8090 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8091 pMixedCtx->rip = IdtEntry.offSel;
8092 pMixedCtx->cs.Sel = IdtEntry.uSel;
8093 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8094 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8095 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8096 && uVector == X86_XCPT_PF)
8097 pMixedCtx->cr2 = GCPtrFaultAddress;
8098
8099 /* If any other guest-state bits are changed here, make sure to update
8100 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8101 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8102 | HM_CHANGED_GUEST_RIP
8103 | HM_CHANGED_GUEST_RFLAGS
8104 | HM_CHANGED_GUEST_RSP);
8105
8106 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8107 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8108 {
8109 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8110 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8111 Log4(("Clearing inhibition due to STI.\n"));
8112 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8113 }
8114 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8115 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8116
8117 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8118 it, if we are returning to ring-3 before executing guest code. */
8119 pVCpu->hm.s.Event.fPending = false;
8120
8121 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8122 if (fStepping)
8123 rcStrict = VINF_EM_DBG_STEPPED;
8124 }
8125 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8126 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8127 return rcStrict;
8128 }
8129
8130 /*
8131 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8132 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8133 */
8134 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8135 }
8136
8137 /* Validate. */
8138 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8139 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8140 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8141
8142 /* Inject. */
8143 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8144 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8145 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8146 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8147
8148 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8149 && uVector == X86_XCPT_PF)
8150 pMixedCtx->cr2 = GCPtrFaultAddress;
8151
8152 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8153 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8154
8155 AssertRCReturn(rc, rc);
8156 return VINF_SUCCESS;
8157}
8158
8159
8160/**
8161 * Clears the interrupt-window exiting control in the VMCS and if necessary
8162 * clears the current event in the VMCS as well.
8163 *
8164 * @returns VBox status code.
8165 * @param pVCpu The cross context virtual CPU structure.
8166 *
8167 * @remarks Use this function only to clear events that have not yet been
8168 * delivered to the guest but are injected in the VMCS!
8169 * @remarks No-long-jump zone!!!
8170 */
8171static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8172{
8173 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8174
8175 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8176 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8177
8178 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8179 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8180}
8181
8182
8183/**
8184 * Enters the VT-x session.
8185 *
8186 * @returns VBox status code.
8187 * @param pVM The cross context VM structure.
8188 * @param pVCpu The cross context virtual CPU structure.
8189 * @param pCpu Pointer to the CPU info struct.
8190 */
8191VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8192{
8193 AssertPtr(pVM);
8194 AssertPtr(pVCpu);
8195 Assert(pVM->hm.s.vmx.fSupported);
8196 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8197 NOREF(pCpu); NOREF(pVM);
8198
8199 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8200 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8201
8202#ifdef VBOX_STRICT
8203 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8204 RTCCUINTREG uHostCR4 = ASMGetCR4();
8205 if (!(uHostCR4 & X86_CR4_VMXE))
8206 {
8207 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8208 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8209 }
8210#endif
8211
8212 /*
8213 * Load the VCPU's VMCS as the current (and active) one.
8214 */
8215 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8216 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8217 if (RT_FAILURE(rc))
8218 return rc;
8219
8220 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8221 pVCpu->hm.s.fLeaveDone = false;
8222 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8223
8224 return VINF_SUCCESS;
8225}
8226
8227
8228/**
8229 * The thread-context callback (only on platforms which support it).
8230 *
8231 * @param enmEvent The thread-context event.
8232 * @param pVCpu The cross context virtual CPU structure.
8233 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8234 * @thread EMT(pVCpu)
8235 */
8236VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8237{
8238 NOREF(fGlobalInit);
8239
8240 switch (enmEvent)
8241 {
8242 case RTTHREADCTXEVENT_OUT:
8243 {
8244 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8245 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8246 VMCPU_ASSERT_EMT(pVCpu);
8247
8248 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8249
8250 /* No longjmps (logger flushes, locks) in this fragile context. */
8251 VMMRZCallRing3Disable(pVCpu);
8252 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8253
8254 /*
8255 * Restore host-state (FPU, debug etc.)
8256 */
8257 if (!pVCpu->hm.s.fLeaveDone)
8258 {
8259 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8260 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8261 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8262 pVCpu->hm.s.fLeaveDone = true;
8263 }
8264
8265 /* Leave HM context, takes care of local init (term). */
8266 int rc = HMR0LeaveCpu(pVCpu);
8267 AssertRC(rc); NOREF(rc);
8268
8269 /* Restore longjmp state. */
8270 VMMRZCallRing3Enable(pVCpu);
8271 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8272 break;
8273 }
8274
8275 case RTTHREADCTXEVENT_IN:
8276 {
8277 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8278 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8279 VMCPU_ASSERT_EMT(pVCpu);
8280
8281 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8282 VMMRZCallRing3Disable(pVCpu);
8283 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8284
8285 /* Initialize the bare minimum state required for HM. This takes care of
8286 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8287 int rc = HMR0EnterCpu(pVCpu);
8288 AssertRC(rc);
8289 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8290
8291 /* Load the active VMCS as the current one. */
8292 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8293 {
8294 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8295 AssertRC(rc); NOREF(rc);
8296 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8297 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8298 }
8299 pVCpu->hm.s.fLeaveDone = false;
8300
8301 /* Restore longjmp state. */
8302 VMMRZCallRing3Enable(pVCpu);
8303 break;
8304 }
8305
8306 default:
8307 break;
8308 }
8309}
8310
8311
8312/**
8313 * Saves the host state in the VMCS host-state.
8314 * Sets up the VM-exit MSR-load area.
8315 *
8316 * The CPU state will be loaded from these fields on every successful VM-exit.
8317 *
8318 * @returns VBox status code.
8319 * @param pVM The cross context VM structure.
8320 * @param pVCpu The cross context virtual CPU structure.
8321 *
8322 * @remarks No-long-jump zone!!!
8323 */
8324static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8325{
8326 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8327
8328 int rc = VINF_SUCCESS;
8329 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8330 {
8331 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8332 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8333
8334 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8335 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8336
8337 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8338 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8339
8340 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8341 }
8342 return rc;
8343}
8344
8345
8346/**
8347 * Saves the host state in the VMCS host-state.
8348 *
8349 * @returns VBox status code.
8350 * @param pVM The cross context VM structure.
8351 * @param pVCpu The cross context virtual CPU structure.
8352 *
8353 * @remarks No-long-jump zone!!!
8354 */
8355VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8356{
8357 AssertPtr(pVM);
8358 AssertPtr(pVCpu);
8359
8360 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8361
8362 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8363 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8364 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8365 return hmR0VmxSaveHostState(pVM, pVCpu);
8366}
8367
8368
8369/**
8370 * Loads the guest state into the VMCS guest-state area.
8371 *
8372 * The will typically be done before VM-entry when the guest-CPU state and the
8373 * VMCS state may potentially be out of sync.
8374 *
8375 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8376 * VM-entry controls.
8377 * Sets up the appropriate VMX non-root function to execute guest code based on
8378 * the guest CPU mode.
8379 *
8380 * @returns VBox strict status code.
8381 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8382 * without unrestricted guest access and the VMMDev is not presently
8383 * mapped (e.g. EFI32).
8384 *
8385 * @param pVM The cross context VM structure.
8386 * @param pVCpu The cross context virtual CPU structure.
8387 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8388 * out-of-sync. Make sure to update the required fields
8389 * before using them.
8390 *
8391 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8392 * caller disables then again on successfull return. Confusing.)
8393 */
8394static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8395{
8396 AssertPtr(pVM);
8397 AssertPtr(pVCpu);
8398 AssertPtr(pMixedCtx);
8399 HMVMX_ASSERT_PREEMPT_SAFE();
8400
8401 VMMRZCallRing3Disable(pVCpu);
8402 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8403
8404 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8405
8406 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8407
8408 /* Determine real-on-v86 mode. */
8409 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8410 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8411 && CPUMIsGuestInRealModeEx(pMixedCtx))
8412 {
8413 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8414 }
8415
8416 /*
8417 * Load the guest-state into the VMCS.
8418 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8419 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8420 */
8421 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8422 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8423
8424 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8425 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8426 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8427
8428 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8429 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8430 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8431
8432 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8433 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8434
8435 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8436 if (rcStrict == VINF_SUCCESS)
8437 { /* likely */ }
8438 else
8439 {
8440 VMMRZCallRing3Enable(pVCpu);
8441 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8442 return rcStrict;
8443 }
8444
8445 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8446 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8447 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8448
8449 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8450 determine we don't have to swap EFER after all. */
8451 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8452 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8453
8454 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8455 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8456
8457 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8458 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8459
8460 /*
8461 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8462 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8463 */
8464 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8465 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8466
8467 /* Clear any unused and reserved bits. */
8468 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8469
8470 VMMRZCallRing3Enable(pVCpu);
8471
8472 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8473 return rc;
8474}
8475
8476
8477/**
8478 * Loads the state shared between the host and guest into the VMCS.
8479 *
8480 * @param pVM The cross context VM structure.
8481 * @param pVCpu The cross context virtual CPU structure.
8482 * @param pCtx Pointer to the guest-CPU context.
8483 *
8484 * @remarks No-long-jump zone!!!
8485 */
8486static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8487{
8488 NOREF(pVM);
8489
8490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8491 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8492
8493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8494 {
8495 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8496 AssertRC(rc);
8497 }
8498
8499 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8500 {
8501 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8502 AssertRC(rc);
8503
8504 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8505 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8506 {
8507 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8508 AssertRC(rc);
8509 }
8510 }
8511
8512 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8513 {
8514 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8515 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8516 }
8517
8518 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8520 {
8521 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8522 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8523 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8524 AssertRC(rc);
8525 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8526 }
8527
8528 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8529 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8530}
8531
8532
8533/**
8534 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8535 *
8536 * @returns Strict VBox status code (i.e. informational status codes too).
8537 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8538 * without unrestricted guest access and the VMMDev is not presently
8539 * mapped (e.g. EFI32).
8540 *
8541 * @param pVM The cross context VM structure.
8542 * @param pVCpu The cross context virtual CPU structure.
8543 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8544 * out-of-sync. Make sure to update the required fields
8545 * before using them.
8546 */
8547static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8548{
8549 HMVMX_ASSERT_PREEMPT_SAFE();
8550
8551 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8552#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8553 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8554#endif
8555
8556 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8557 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8558 {
8559 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8560 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8561 { /* likely */}
8562 else
8563 {
8564 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8565 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8566 }
8567 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8568 }
8569 else if (HMCPU_CF_VALUE(pVCpu))
8570 {
8571 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8572 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8573 { /* likely */}
8574 else
8575 {
8576 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8577 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8578 return rcStrict;
8579 }
8580 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8581 }
8582
8583 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8584 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8585 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8586 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8587 return rcStrict;
8588}
8589
8590
8591/**
8592 * Does the preparations before executing guest code in VT-x.
8593 *
8594 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8595 * recompiler/IEM. We must be cautious what we do here regarding committing
8596 * guest-state information into the VMCS assuming we assuredly execute the
8597 * guest in VT-x mode.
8598 *
8599 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8600 * the common-state (TRPM/forceflags), we must undo those changes so that the
8601 * recompiler/IEM can (and should) use them when it resumes guest execution.
8602 * Otherwise such operations must be done when we can no longer exit to ring-3.
8603 *
8604 * @returns Strict VBox status code (i.e. informational status codes too).
8605 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8606 * have been disabled.
8607 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8608 * double-fault into the guest.
8609 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8610 * dispatched directly.
8611 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8612 *
8613 * @param pVM The cross context VM structure.
8614 * @param pVCpu The cross context virtual CPU structure.
8615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8616 * out-of-sync. Make sure to update the required fields
8617 * before using them.
8618 * @param pVmxTransient Pointer to the VMX transient structure.
8619 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8620 * us ignore some of the reasons for returning to
8621 * ring-3, and return VINF_EM_DBG_STEPPED if event
8622 * dispatching took place.
8623 */
8624static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8625{
8626 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8627
8628#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8629 PGMRZDynMapFlushAutoSet(pVCpu);
8630#endif
8631
8632 /* Check force flag actions that might require us to go back to ring-3. */
8633 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8634 if (rcStrict == VINF_SUCCESS)
8635 { /* FFs doesn't get set all the time. */ }
8636 else
8637 return rcStrict;
8638
8639 if (TRPMHasTrap(pVCpu))
8640 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8641 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8642
8643 /*
8644 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8645 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8646 */
8647 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8648 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8649 { /* likely */ }
8650 else
8651 {
8652 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8653 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8654 return rcStrict;
8655 }
8656
8657 /*
8658 * Load the guest state bits, we can handle longjmps/getting preempted here.
8659 *
8660 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8661 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8662 * Hence, this needs to be done -after- injection of events.
8663 */
8664 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8665 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8666 { /* likely */ }
8667 else
8668 return rcStrict;
8669
8670 /*
8671 * No longjmps to ring-3 from this point on!!!
8672 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8673 * This also disables flushing of the R0-logger instance (if any).
8674 */
8675 VMMRZCallRing3Disable(pVCpu);
8676
8677 /*
8678 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8679 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8680 *
8681 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8682 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8683 *
8684 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8685 * executing guest code.
8686 */
8687 pVmxTransient->fEFlags = ASMIntDisableFlags();
8688
8689 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8690 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8691 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8692 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8693 {
8694 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8695 {
8696 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8697 pVCpu->hm.s.Event.fPending = false;
8698
8699 return VINF_SUCCESS;
8700 }
8701
8702 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8703 rcStrict = VINF_EM_RAW_INTERRUPT;
8704 }
8705 else
8706 {
8707 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8708 rcStrict = VINF_EM_RAW_TO_R3;
8709 }
8710
8711 ASMSetFlags(pVmxTransient->fEFlags);
8712 VMMRZCallRing3Enable(pVCpu);
8713
8714 return rcStrict;
8715}
8716
8717
8718/**
8719 * Prepares to run guest code in VT-x and we've committed to doing so. This
8720 * means there is no backing out to ring-3 or anywhere else at this
8721 * point.
8722 *
8723 * @param pVM The cross context VM structure.
8724 * @param pVCpu The cross context virtual CPU structure.
8725 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8726 * out-of-sync. Make sure to update the required fields
8727 * before using them.
8728 * @param pVmxTransient Pointer to the VMX transient structure.
8729 *
8730 * @remarks Called with preemption disabled.
8731 * @remarks No-long-jump zone!!!
8732 */
8733static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8734{
8735 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8736 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8737 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8738
8739 /*
8740 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8741 */
8742 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8743 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8744
8745#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8746 if (!CPUMIsGuestFPUStateActive(pVCpu))
8747 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8748 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8749 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8750#endif
8751
8752 if ( pVCpu->hm.s.fPreloadGuestFpu
8753 && !CPUMIsGuestFPUStateActive(pVCpu))
8754 {
8755 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8756 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8757 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8758 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8759 }
8760
8761 /*
8762 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8763 */
8764 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8765 && pVCpu->hm.s.vmx.cMsrs > 0)
8766 {
8767 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8768 }
8769
8770 /*
8771 * Load the host state bits as we may've been preempted (only happens when
8772 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8773 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8774 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8775 * See @bugref{8432}.
8776 */
8777 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8778 {
8779 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8780 AssertRC(rc);
8781 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8782 }
8783 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8784
8785 /*
8786 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8787 */
8788 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8789 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8790 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8791
8792 /* Store status of the shared guest-host state at the time of VM-entry. */
8793#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8794 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8795 {
8796 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8797 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8798 }
8799 else
8800#endif
8801 {
8802 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8803 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8804 }
8805 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8806
8807 /*
8808 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8809 */
8810 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8811 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8812
8813 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8814 RTCPUID idCurrentCpu = pCpu->idCpu;
8815 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8816 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8817 {
8818 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8819 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8820 }
8821
8822 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8823 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8824 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8825 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8826
8827 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8828
8829 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8830 to start executing. */
8831
8832 /*
8833 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8834 */
8835 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8836 {
8837 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8838 {
8839 bool fMsrUpdated;
8840 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8841 AssertRC(rc2);
8842 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8843
8844 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8845 &fMsrUpdated);
8846 AssertRC(rc2);
8847 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8848
8849 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8850 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8851 }
8852 else
8853 {
8854 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8855 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8856 }
8857 }
8858
8859#ifdef VBOX_STRICT
8860 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8861 hmR0VmxCheckHostEferMsr(pVCpu);
8862 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8863#endif
8864#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8865 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8866 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8867 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8868#endif
8869}
8870
8871
8872/**
8873 * Performs some essential restoration of state after running guest code in
8874 * VT-x.
8875 *
8876 * @param pVM The cross context VM structure.
8877 * @param pVCpu The cross context virtual CPU structure.
8878 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8879 * out-of-sync. Make sure to update the required fields
8880 * before using them.
8881 * @param pVmxTransient Pointer to the VMX transient structure.
8882 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8883 *
8884 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8885 *
8886 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8887 * unconditionally when it is safe to do so.
8888 */
8889static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8890{
8891 NOREF(pVM);
8892
8893 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8894
8895 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8896 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8897 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8898 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8899 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8900 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8901
8902 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8903 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8904
8905 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8906 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8907 Assert(!ASMIntAreEnabled());
8908 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8909
8910#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8911 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8912 {
8913 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8914 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8915 }
8916#endif
8917
8918#if HC_ARCH_BITS == 64
8919 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8920#endif
8921#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8922 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8923 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8924 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8925#else
8926 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8927#endif
8928#ifdef VBOX_STRICT
8929 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8930#endif
8931 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8932 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8933
8934 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8935 uint32_t uExitReason;
8936 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8937 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8938 AssertRC(rc);
8939 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8940 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8941
8942 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8943 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8944 {
8945 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8946 pVmxTransient->fVMEntryFailed));
8947 return;
8948 }
8949
8950 /*
8951 * Update the VM-exit history array here even if the VM-entry failed due to:
8952 * - Invalid guest state.
8953 * - MSR loading.
8954 * - Machine-check event.
8955 *
8956 * In any of the above cases we will still have a "valid" VM-exit reason
8957 * despite @a fVMEntryFailed being false.
8958 *
8959 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8960 */
8961 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8962
8963 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8964 {
8965 /** @todo We can optimize this by only syncing with our force-flags when
8966 * really needed and keeping the VMCS state as it is for most
8967 * VM-exits. */
8968 /* Update the guest interruptibility-state from the VMCS. */
8969 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8970
8971#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8972 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8973 AssertRC(rc);
8974#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8975 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8976 AssertRC(rc);
8977#endif
8978
8979 /*
8980 * Sync the TPR shadow with our APIC state.
8981 */
8982 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8983 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8984 {
8985 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8986 AssertRC(rc);
8987 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8988 }
8989 }
8990}
8991
8992
8993/**
8994 * Runs the guest code using VT-x the normal way.
8995 *
8996 * @returns VBox status code.
8997 * @param pVM The cross context VM structure.
8998 * @param pVCpu The cross context virtual CPU structure.
8999 * @param pCtx Pointer to the guest-CPU context.
9000 *
9001 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9002 */
9003static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9004{
9005 VMXTRANSIENT VmxTransient;
9006 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9007 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9008 uint32_t cLoops = 0;
9009
9010 for (;; cLoops++)
9011 {
9012 Assert(!HMR0SuspendPending());
9013 HMVMX_ASSERT_CPU_SAFE();
9014
9015 /* Preparatory work for running guest code, this may force us to return
9016 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9017 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9018 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9019 if (rcStrict != VINF_SUCCESS)
9020 break;
9021
9022 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9023 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9024 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9025
9026 /* Restore any residual host-state and save any bits shared between host
9027 and guest into the guest-CPU state. Re-enables interrupts! */
9028 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9029
9030 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9031 if (RT_SUCCESS(rcRun))
9032 { /* very likely */ }
9033 else
9034 {
9035 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9036 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9037 return rcRun;
9038 }
9039
9040 /* Profile the VM-exit. */
9041 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9042 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9043 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9044 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9045 HMVMX_START_EXIT_DISPATCH_PROF();
9046
9047 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9048
9049 /* Handle the VM-exit. */
9050#ifdef HMVMX_USE_FUNCTION_TABLE
9051 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9052#else
9053 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9054#endif
9055 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9056 if (rcStrict == VINF_SUCCESS)
9057 {
9058 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9059 continue; /* likely */
9060 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9061 rcStrict = VINF_EM_RAW_INTERRUPT;
9062 }
9063 break;
9064 }
9065
9066 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9067 return rcStrict;
9068}
9069
9070
9071
9072/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9073 * probes.
9074 *
9075 * The following few functions and associated structure contains the bloat
9076 * necessary for providing detailed debug events and dtrace probes as well as
9077 * reliable host side single stepping. This works on the principle of
9078 * "subclassing" the normal execution loop and workers. We replace the loop
9079 * method completely and override selected helpers to add necessary adjustments
9080 * to their core operation.
9081 *
9082 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9083 * any performance for debug and analysis features.
9084 *
9085 * @{
9086 */
9087
9088/**
9089 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9090 * the debug run loop.
9091 */
9092typedef struct VMXRUNDBGSTATE
9093{
9094 /** The RIP we started executing at. This is for detecting that we stepped. */
9095 uint64_t uRipStart;
9096 /** The CS we started executing with. */
9097 uint16_t uCsStart;
9098
9099 /** Whether we've actually modified the 1st execution control field. */
9100 bool fModifiedProcCtls : 1;
9101 /** Whether we've actually modified the 2nd execution control field. */
9102 bool fModifiedProcCtls2 : 1;
9103 /** Whether we've actually modified the exception bitmap. */
9104 bool fModifiedXcptBitmap : 1;
9105
9106 /** We desire the modified the CR0 mask to be cleared. */
9107 bool fClearCr0Mask : 1;
9108 /** We desire the modified the CR4 mask to be cleared. */
9109 bool fClearCr4Mask : 1;
9110 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9111 uint32_t fCpe1Extra;
9112 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9113 uint32_t fCpe1Unwanted;
9114 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9115 uint32_t fCpe2Extra;
9116 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9117 uint32_t bmXcptExtra;
9118 /** The sequence number of the Dtrace provider settings the state was
9119 * configured against. */
9120 uint32_t uDtraceSettingsSeqNo;
9121 /** VM-exits to check (one bit per VM-exit). */
9122 uint32_t bmExitsToCheck[3];
9123
9124 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9125 uint32_t fProcCtlsInitial;
9126 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9127 uint32_t fProcCtls2Initial;
9128 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9129 uint32_t bmXcptInitial;
9130} VMXRUNDBGSTATE;
9131AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9132typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9133
9134
9135/**
9136 * Initializes the VMXRUNDBGSTATE structure.
9137 *
9138 * @param pVCpu The cross context virtual CPU structure of the
9139 * calling EMT.
9140 * @param pCtx The CPU register context to go with @a pVCpu.
9141 * @param pDbgState The structure to initialize.
9142 */
9143DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9144{
9145 pDbgState->uRipStart = pCtx->rip;
9146 pDbgState->uCsStart = pCtx->cs.Sel;
9147
9148 pDbgState->fModifiedProcCtls = false;
9149 pDbgState->fModifiedProcCtls2 = false;
9150 pDbgState->fModifiedXcptBitmap = false;
9151 pDbgState->fClearCr0Mask = false;
9152 pDbgState->fClearCr4Mask = false;
9153 pDbgState->fCpe1Extra = 0;
9154 pDbgState->fCpe1Unwanted = 0;
9155 pDbgState->fCpe2Extra = 0;
9156 pDbgState->bmXcptExtra = 0;
9157 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9158 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9159 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9160}
9161
9162
9163/**
9164 * Updates the VMSC fields with changes requested by @a pDbgState.
9165 *
9166 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9167 * immediately before executing guest code, i.e. when interrupts are disabled.
9168 * We don't check status codes here as we cannot easily assert or return in the
9169 * latter case.
9170 *
9171 * @param pVCpu The cross context virtual CPU structure.
9172 * @param pDbgState The debug state.
9173 */
9174DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9175{
9176 /*
9177 * Ensure desired flags in VMCS control fields are set.
9178 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9179 *
9180 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9181 * there should be no stale data in pCtx at this point.
9182 */
9183 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9184 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9185 {
9186 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9187 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9188 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9189 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9190 pDbgState->fModifiedProcCtls = true;
9191 }
9192
9193 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9194 {
9195 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9196 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9197 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9198 pDbgState->fModifiedProcCtls2 = true;
9199 }
9200
9201 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9202 {
9203 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9204 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9205 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9206 pDbgState->fModifiedXcptBitmap = true;
9207 }
9208
9209 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9210 {
9211 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9212 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9213 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9214 }
9215
9216 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9217 {
9218 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9219 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9220 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9221 }
9222}
9223
9224
9225DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9226{
9227 /*
9228 * Restore VM-exit control settings as we may not reenter this function the
9229 * next time around.
9230 */
9231 /* We reload the initial value, trigger what we can of recalculations the
9232 next time around. From the looks of things, that's all that's required atm. */
9233 if (pDbgState->fModifiedProcCtls)
9234 {
9235 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9236 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9237 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9238 AssertRCReturn(rc2, rc2);
9239 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9240 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9241 }
9242
9243 /* We're currently the only ones messing with this one, so just restore the
9244 cached value and reload the field. */
9245 if ( pDbgState->fModifiedProcCtls2
9246 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9247 {
9248 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9249 AssertRCReturn(rc2, rc2);
9250 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9251 }
9252
9253 /* If we've modified the exception bitmap, we restore it and trigger
9254 reloading and partial recalculation the next time around. */
9255 if (pDbgState->fModifiedXcptBitmap)
9256 {
9257 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9258 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9259 }
9260
9261 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9262 if (pDbgState->fClearCr0Mask)
9263 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9264
9265 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9266 if (pDbgState->fClearCr4Mask)
9267 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9268
9269 return rcStrict;
9270}
9271
9272
9273/**
9274 * Configures VM-exit controls for current DBGF and DTrace settings.
9275 *
9276 * This updates @a pDbgState and the VMCS execution control fields to reflect
9277 * the necessary VM-exits demanded by DBGF and DTrace.
9278 *
9279 * @param pVM The cross context VM structure.
9280 * @param pVCpu The cross context virtual CPU structure.
9281 * @param pCtx Pointer to the guest-CPU context.
9282 * @param pDbgState The debug state.
9283 * @param pVmxTransient Pointer to the VMX transient structure. May update
9284 * fUpdateTscOffsettingAndPreemptTimer.
9285 */
9286static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9287 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9288{
9289 /*
9290 * Take down the dtrace serial number so we can spot changes.
9291 */
9292 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9293 ASMCompilerBarrier();
9294
9295 /*
9296 * We'll rebuild most of the middle block of data members (holding the
9297 * current settings) as we go along here, so start by clearing it all.
9298 */
9299 pDbgState->bmXcptExtra = 0;
9300 pDbgState->fCpe1Extra = 0;
9301 pDbgState->fCpe1Unwanted = 0;
9302 pDbgState->fCpe2Extra = 0;
9303 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9304 pDbgState->bmExitsToCheck[i] = 0;
9305
9306 /*
9307 * Software interrupts (INT XXh) - no idea how to trigger these...
9308 */
9309 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9310 || VBOXVMM_INT_SOFTWARE_ENABLED())
9311 {
9312 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9313 }
9314
9315 /*
9316 * INT3 breakpoints - triggered by #BP exceptions.
9317 */
9318 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9319 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9320
9321 /*
9322 * Exception bitmap and XCPT events+probes.
9323 */
9324 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9325 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9326 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9327
9328 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9329 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9330 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9331 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9332 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9333 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9334 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9335 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9336 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9337 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9338 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9339 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9340 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9341 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9342 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9343 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9344 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9345 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9346
9347 if (pDbgState->bmXcptExtra)
9348 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9349
9350 /*
9351 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9352 *
9353 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9354 * So, when adding/changing/removing please don't forget to update it.
9355 *
9356 * Some of the macros are picking up local variables to save horizontal space,
9357 * (being able to see it in a table is the lesser evil here).
9358 */
9359#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9360 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9361 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9362#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9363 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9364 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9365 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9366 } else do { } while (0)
9367#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9368 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9369 { \
9370 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9371 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9372 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9373 } else do { } while (0)
9374#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9375 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9376 { \
9377 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9378 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9379 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9380 } else do { } while (0)
9381#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9382 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9383 { \
9384 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9385 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9386 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9387 } else do { } while (0)
9388
9389 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9390 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9391 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9392 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9393 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9394
9395 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9397 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9398 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9399 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9400 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9401 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9403 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9404 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9405 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9406 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9407 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9408 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9409 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9411 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9412 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9413 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9415 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9417 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9419 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9421 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9423 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9425 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9431
9432 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9433 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9434 {
9435 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9436 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9437 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9438 AssertRC(rc2);
9439
9440#if 0 /** @todo fix me */
9441 pDbgState->fClearCr0Mask = true;
9442 pDbgState->fClearCr4Mask = true;
9443#endif
9444 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9445 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9446 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9447 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9448 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9449 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9450 require clearing here and in the loop if we start using it. */
9451 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9452 }
9453 else
9454 {
9455 if (pDbgState->fClearCr0Mask)
9456 {
9457 pDbgState->fClearCr0Mask = false;
9458 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9459 }
9460 if (pDbgState->fClearCr4Mask)
9461 {
9462 pDbgState->fClearCr4Mask = false;
9463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9464 }
9465 }
9466 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9467 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9468
9469 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9470 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9471 {
9472 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9473 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9474 }
9475 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9476 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9477
9478 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9479 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9480 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9482 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9483 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9484 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9485 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9486#if 0 /** @todo too slow, fix handler. */
9487 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9488#endif
9489 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9490
9491 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9492 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9493 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9494 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9495 {
9496 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9497 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9498 }
9499 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9500 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9501 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9502 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9503
9504 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9505 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9506 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9507 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9508 {
9509 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9510 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9511 }
9512 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9513 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9514 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9515 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9516
9517 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9518 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9519 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9520 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9521 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9522 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9523 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9524 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9525 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9526 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9527 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9528 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9529 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9530 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9531 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9532 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9533 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9535 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9536 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9537 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9538 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9539
9540#undef IS_EITHER_ENABLED
9541#undef SET_ONLY_XBM_IF_EITHER_EN
9542#undef SET_CPE1_XBM_IF_EITHER_EN
9543#undef SET_CPEU_XBM_IF_EITHER_EN
9544#undef SET_CPE2_XBM_IF_EITHER_EN
9545
9546 /*
9547 * Sanitize the control stuff.
9548 */
9549 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9550 if (pDbgState->fCpe2Extra)
9551 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9552 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9553 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9554 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9555 {
9556 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9557 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9558 }
9559
9560 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9561 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9562 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9563 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9564}
9565
9566
9567/**
9568 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9569 * appropriate.
9570 *
9571 * The caller has checked the VM-exit against the
9572 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9573 * already, so we don't have to do that either.
9574 *
9575 * @returns Strict VBox status code (i.e. informational status codes too).
9576 * @param pVM The cross context VM structure.
9577 * @param pVCpu The cross context virtual CPU structure.
9578 * @param pMixedCtx Pointer to the guest-CPU context.
9579 * @param pVmxTransient Pointer to the VMX-transient structure.
9580 * @param uExitReason The VM-exit reason.
9581 *
9582 * @remarks The name of this function is displayed by dtrace, so keep it short
9583 * and to the point. No longer than 33 chars long, please.
9584 */
9585static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9586 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9587{
9588 /*
9589 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9590 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9591 *
9592 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9593 * does. Must add/change/remove both places. Same ordering, please.
9594 *
9595 * Added/removed events must also be reflected in the next section
9596 * where we dispatch dtrace events.
9597 */
9598 bool fDtrace1 = false;
9599 bool fDtrace2 = false;
9600 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9601 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9602 uint32_t uEventArg = 0;
9603#define SET_EXIT(a_EventSubName) \
9604 do { \
9605 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9606 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9607 } while (0)
9608#define SET_BOTH(a_EventSubName) \
9609 do { \
9610 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9611 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9612 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9613 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9614 } while (0)
9615 switch (uExitReason)
9616 {
9617 case VMX_EXIT_MTF:
9618 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9619
9620 case VMX_EXIT_XCPT_OR_NMI:
9621 {
9622 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9623 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9624 {
9625 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9626 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9627 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9628 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9629 {
9630 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9631 {
9632 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9633 uEventArg = pVmxTransient->uExitIntErrorCode;
9634 }
9635 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9636 switch (enmEvent1)
9637 {
9638 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9639 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9640 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9641 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9642 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9643 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9644 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9645 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9646 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9647 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9648 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9649 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9650 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9651 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9652 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9653 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9654 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9655 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9656 default: break;
9657 }
9658 }
9659 else
9660 AssertFailed();
9661 break;
9662
9663 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9664 uEventArg = idxVector;
9665 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9666 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9667 break;
9668 }
9669 break;
9670 }
9671
9672 case VMX_EXIT_TRIPLE_FAULT:
9673 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9674 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9675 break;
9676 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9677 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9678 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9679 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9680 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9681
9682 /* Instruction specific VM-exits: */
9683 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9684 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9685 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9686 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9687 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9688 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9689 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9690 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9691 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9692 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9693 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9694 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9695 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9696 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9697 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9698 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9699 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9700 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9701 case VMX_EXIT_MOV_CRX:
9702 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9703/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9704* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9705 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9706 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9707 SET_BOTH(CRX_READ);
9708 else
9709 SET_BOTH(CRX_WRITE);
9710 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9711 break;
9712 case VMX_EXIT_MOV_DRX:
9713 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9714 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9715 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9716 SET_BOTH(DRX_READ);
9717 else
9718 SET_BOTH(DRX_WRITE);
9719 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9720 break;
9721 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9722 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9723 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9724 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9725 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9726 case VMX_EXIT_XDTR_ACCESS:
9727 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9728 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9729 {
9730 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9731 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9732 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9733 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9734 }
9735 break;
9736
9737 case VMX_EXIT_TR_ACCESS:
9738 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9739 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9740 {
9741 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9742 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9743 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9744 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9745 }
9746 break;
9747
9748 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9749 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9750 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9751 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9752 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9753 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9754 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9755 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9756 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9757 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9758 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9759
9760 /* Events that aren't relevant at this point. */
9761 case VMX_EXIT_EXT_INT:
9762 case VMX_EXIT_INT_WINDOW:
9763 case VMX_EXIT_NMI_WINDOW:
9764 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9765 case VMX_EXIT_PREEMPT_TIMER:
9766 case VMX_EXIT_IO_INSTR:
9767 break;
9768
9769 /* Errors and unexpected events. */
9770 case VMX_EXIT_INIT_SIGNAL:
9771 case VMX_EXIT_SIPI:
9772 case VMX_EXIT_IO_SMI:
9773 case VMX_EXIT_SMI:
9774 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9775 case VMX_EXIT_ERR_MSR_LOAD:
9776 case VMX_EXIT_ERR_MACHINE_CHECK:
9777 break;
9778
9779 default:
9780 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9781 break;
9782 }
9783#undef SET_BOTH
9784#undef SET_EXIT
9785
9786 /*
9787 * Dtrace tracepoints go first. We do them here at once so we don't
9788 * have to copy the guest state saving and stuff a few dozen times.
9789 * Down side is that we've got to repeat the switch, though this time
9790 * we use enmEvent since the probes are a subset of what DBGF does.
9791 */
9792 if (fDtrace1 || fDtrace2)
9793 {
9794 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9795 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9796 switch (enmEvent1)
9797 {
9798 /** @todo consider which extra parameters would be helpful for each probe. */
9799 case DBGFEVENT_END: break;
9800 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9802 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9809 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9810 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9811 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9812 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9813 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9818 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9819 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9820 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9828 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9829 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9830 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9831 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9832 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9833 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9834 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9836 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9839 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9844 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9845 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9866 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9867 }
9868 switch (enmEvent2)
9869 {
9870 /** @todo consider which extra parameters would be helpful for each probe. */
9871 case DBGFEVENT_END: break;
9872 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9874 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9882 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9883 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9884 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9885 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9886 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9887 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9888 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9897 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9898 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9899 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9900 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9901 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9902 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9903 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9904 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9905 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9906 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9908 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9924 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9925 }
9926 }
9927
9928 /*
9929 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9930 * the DBGF call will do a full check).
9931 *
9932 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9933 * Note! If we have to events, we prioritize the first, i.e. the instruction
9934 * one, in order to avoid event nesting.
9935 */
9936 if ( enmEvent1 != DBGFEVENT_END
9937 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9938 {
9939 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9940 if (rcStrict != VINF_SUCCESS)
9941 return rcStrict;
9942 }
9943 else if ( enmEvent2 != DBGFEVENT_END
9944 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9945 {
9946 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9947 if (rcStrict != VINF_SUCCESS)
9948 return rcStrict;
9949 }
9950
9951 return VINF_SUCCESS;
9952}
9953
9954
9955/**
9956 * Single-stepping VM-exit filtering.
9957 *
9958 * This is preprocessing the VM-exits and deciding whether we've gotten far
9959 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9960 * handling is performed.
9961 *
9962 * @returns Strict VBox status code (i.e. informational status codes too).
9963 * @param pVM The cross context VM structure.
9964 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9965 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9966 * out-of-sync. Make sure to update the required
9967 * fields before using them.
9968 * @param pVmxTransient Pointer to the VMX-transient structure.
9969 * @param uExitReason The VM-exit reason.
9970 * @param pDbgState The debug state.
9971 */
9972DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9973 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9974{
9975 /*
9976 * Expensive (saves context) generic dtrace VM-exit probe.
9977 */
9978 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9979 { /* more likely */ }
9980 else
9981 {
9982 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9983 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9984 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9985 }
9986
9987 /*
9988 * Check for host NMI, just to get that out of the way.
9989 */
9990 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9991 { /* normally likely */ }
9992 else
9993 {
9994 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9995 AssertRCReturn(rc2, rc2);
9996 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9997 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9998 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9999 }
10000
10001 /*
10002 * Check for single stepping event if we're stepping.
10003 */
10004 if (pVCpu->hm.s.fSingleInstruction)
10005 {
10006 switch (uExitReason)
10007 {
10008 case VMX_EXIT_MTF:
10009 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10010
10011 /* Various events: */
10012 case VMX_EXIT_XCPT_OR_NMI:
10013 case VMX_EXIT_EXT_INT:
10014 case VMX_EXIT_TRIPLE_FAULT:
10015 case VMX_EXIT_INT_WINDOW:
10016 case VMX_EXIT_NMI_WINDOW:
10017 case VMX_EXIT_TASK_SWITCH:
10018 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10019 case VMX_EXIT_APIC_ACCESS:
10020 case VMX_EXIT_EPT_VIOLATION:
10021 case VMX_EXIT_EPT_MISCONFIG:
10022 case VMX_EXIT_PREEMPT_TIMER:
10023
10024 /* Instruction specific VM-exits: */
10025 case VMX_EXIT_CPUID:
10026 case VMX_EXIT_GETSEC:
10027 case VMX_EXIT_HLT:
10028 case VMX_EXIT_INVD:
10029 case VMX_EXIT_INVLPG:
10030 case VMX_EXIT_RDPMC:
10031 case VMX_EXIT_RDTSC:
10032 case VMX_EXIT_RSM:
10033 case VMX_EXIT_VMCALL:
10034 case VMX_EXIT_VMCLEAR:
10035 case VMX_EXIT_VMLAUNCH:
10036 case VMX_EXIT_VMPTRLD:
10037 case VMX_EXIT_VMPTRST:
10038 case VMX_EXIT_VMREAD:
10039 case VMX_EXIT_VMRESUME:
10040 case VMX_EXIT_VMWRITE:
10041 case VMX_EXIT_VMXOFF:
10042 case VMX_EXIT_VMXON:
10043 case VMX_EXIT_MOV_CRX:
10044 case VMX_EXIT_MOV_DRX:
10045 case VMX_EXIT_IO_INSTR:
10046 case VMX_EXIT_RDMSR:
10047 case VMX_EXIT_WRMSR:
10048 case VMX_EXIT_MWAIT:
10049 case VMX_EXIT_MONITOR:
10050 case VMX_EXIT_PAUSE:
10051 case VMX_EXIT_XDTR_ACCESS:
10052 case VMX_EXIT_TR_ACCESS:
10053 case VMX_EXIT_INVEPT:
10054 case VMX_EXIT_RDTSCP:
10055 case VMX_EXIT_INVVPID:
10056 case VMX_EXIT_WBINVD:
10057 case VMX_EXIT_XSETBV:
10058 case VMX_EXIT_RDRAND:
10059 case VMX_EXIT_INVPCID:
10060 case VMX_EXIT_VMFUNC:
10061 case VMX_EXIT_RDSEED:
10062 case VMX_EXIT_XSAVES:
10063 case VMX_EXIT_XRSTORS:
10064 {
10065 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10066 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10067 AssertRCReturn(rc2, rc2);
10068 if ( pMixedCtx->rip != pDbgState->uRipStart
10069 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10070 return VINF_EM_DBG_STEPPED;
10071 break;
10072 }
10073
10074 /* Errors and unexpected events: */
10075 case VMX_EXIT_INIT_SIGNAL:
10076 case VMX_EXIT_SIPI:
10077 case VMX_EXIT_IO_SMI:
10078 case VMX_EXIT_SMI:
10079 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10080 case VMX_EXIT_ERR_MSR_LOAD:
10081 case VMX_EXIT_ERR_MACHINE_CHECK:
10082 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10083 break;
10084
10085 default:
10086 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10087 break;
10088 }
10089 }
10090
10091 /*
10092 * Check for debugger event breakpoints and dtrace probes.
10093 */
10094 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10095 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10096 {
10097 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10098 if (rcStrict != VINF_SUCCESS)
10099 return rcStrict;
10100 }
10101
10102 /*
10103 * Normal processing.
10104 */
10105#ifdef HMVMX_USE_FUNCTION_TABLE
10106 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10107#else
10108 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10109#endif
10110}
10111
10112
10113/**
10114 * Single steps guest code using VT-x.
10115 *
10116 * @returns Strict VBox status code (i.e. informational status codes too).
10117 * @param pVM The cross context VM structure.
10118 * @param pVCpu The cross context virtual CPU structure.
10119 * @param pCtx Pointer to the guest-CPU context.
10120 *
10121 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10122 */
10123static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10124{
10125 VMXTRANSIENT VmxTransient;
10126 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10127
10128 /* Set HMCPU indicators. */
10129 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10130 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10131 pVCpu->hm.s.fDebugWantRdTscExit = false;
10132 pVCpu->hm.s.fUsingDebugLoop = true;
10133
10134 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10135 VMXRUNDBGSTATE DbgState;
10136 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10137 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10138
10139 /*
10140 * The loop.
10141 */
10142 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10143 for (uint32_t cLoops = 0; ; cLoops++)
10144 {
10145 Assert(!HMR0SuspendPending());
10146 HMVMX_ASSERT_CPU_SAFE();
10147 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10148
10149 /*
10150 * Preparatory work for running guest code, this may force us to return
10151 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10152 */
10153 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10154 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10155 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10156 if (rcStrict != VINF_SUCCESS)
10157 break;
10158
10159 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10160 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10161
10162 /*
10163 * Now we can run the guest code.
10164 */
10165 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10166
10167 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10168
10169 /*
10170 * Restore any residual host-state and save any bits shared between host
10171 * and guest into the guest-CPU state. Re-enables interrupts!
10172 */
10173 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10174
10175 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10176 if (RT_SUCCESS(rcRun))
10177 { /* very likely */ }
10178 else
10179 {
10180 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10181 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10182 return rcRun;
10183 }
10184
10185 /* Profile the VM-exit. */
10186 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10188 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10189 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10190 HMVMX_START_EXIT_DISPATCH_PROF();
10191
10192 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10193
10194 /*
10195 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10196 */
10197 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10198 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10199 if (rcStrict != VINF_SUCCESS)
10200 break;
10201 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10202 {
10203 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10204 rcStrict = VINF_EM_RAW_INTERRUPT;
10205 break;
10206 }
10207
10208 /*
10209 * Stepping: Did the RIP change, if so, consider it a single step.
10210 * Otherwise, make sure one of the TFs gets set.
10211 */
10212 if (fStepping)
10213 {
10214 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10215 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10216 AssertRCReturn(rc2, rc2);
10217 if ( pCtx->rip != DbgState.uRipStart
10218 || pCtx->cs.Sel != DbgState.uCsStart)
10219 {
10220 rcStrict = VINF_EM_DBG_STEPPED;
10221 break;
10222 }
10223 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10224 }
10225
10226 /*
10227 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10228 */
10229 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10230 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10231 }
10232
10233 /*
10234 * Clear the X86_EFL_TF if necessary.
10235 */
10236 if (pVCpu->hm.s.fClearTrapFlag)
10237 {
10238 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10239 AssertRCReturn(rc2, rc2);
10240 pVCpu->hm.s.fClearTrapFlag = false;
10241 pCtx->eflags.Bits.u1TF = 0;
10242 }
10243 /** @todo there seems to be issues with the resume flag when the monitor trap
10244 * flag is pending without being used. Seen early in bios init when
10245 * accessing APIC page in protected mode. */
10246
10247 /*
10248 * Restore VM-exit control settings as we may not reenter this function the
10249 * next time around.
10250 */
10251 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10252
10253 /* Restore HMCPU indicators. */
10254 pVCpu->hm.s.fUsingDebugLoop = false;
10255 pVCpu->hm.s.fDebugWantRdTscExit = false;
10256 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10257
10258 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10259 return rcStrict;
10260}
10261
10262
10263/** @} */
10264
10265
10266/**
10267 * Checks if any expensive dtrace probes are enabled and we should go to the
10268 * debug loop.
10269 *
10270 * @returns true if we should use debug loop, false if not.
10271 */
10272static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10273{
10274 /* It's probably faster to OR the raw 32-bit counter variables together.
10275 Since the variables are in an array and the probes are next to one
10276 another (more or less), we have good locality. So, better read
10277 eight-nine cache lines ever time and only have one conditional, than
10278 128+ conditionals, right? */
10279 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10280 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10281 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10282 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10283 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10284 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10285 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10286 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10287 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10288 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10289 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10290 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10291 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10292 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10293 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10294 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10295 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10296 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10297 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10298 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10299 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10300 ) != 0
10301 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10302 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10303 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10304 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10305 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10306 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10307 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10308 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10309 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10310 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10311 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10312 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10313 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10314 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10315 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10316 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10317 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10318 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10319 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10320 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10321 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10322 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10323 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10324 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10325 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10326 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10327 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10328 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10329 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10330 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10331 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10332 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10333 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10334 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10335 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10336 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10337 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10338 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10339 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10340 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10341 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10342 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10343 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10344 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10345 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10346 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10347 ) != 0
10348 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10349 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10350 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10351 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10352 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10353 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10354 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10355 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10356 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10357 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10358 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10359 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10360 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10361 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10362 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10363 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10364 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10365 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10366 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10367 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10368 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10369 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10370 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10371 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10372 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10373 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10374 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10375 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10376 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10377 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10378 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10379 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10380 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10381 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10382 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10383 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10384 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10385 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10386 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10387 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10388 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10389 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10390 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10391 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10392 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10393 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10394 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10395 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10396 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10397 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10398 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10399 ) != 0;
10400}
10401
10402
10403/**
10404 * Runs the guest code using VT-x.
10405 *
10406 * @returns Strict VBox status code (i.e. informational status codes too).
10407 * @param pVM The cross context VM structure.
10408 * @param pVCpu The cross context virtual CPU structure.
10409 * @param pCtx Pointer to the guest-CPU context.
10410 */
10411VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10412{
10413 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10414 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10415 HMVMX_ASSERT_PREEMPT_SAFE();
10416
10417 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10418
10419 VBOXSTRICTRC rcStrict;
10420 if ( !pVCpu->hm.s.fUseDebugLoop
10421 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10422 && !DBGFIsStepping(pVCpu)
10423 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10424 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10425 else
10426 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10427
10428 if (rcStrict == VERR_EM_INTERPRETER)
10429 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10430 else if (rcStrict == VINF_EM_RESET)
10431 rcStrict = VINF_EM_TRIPLE_FAULT;
10432
10433 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10434 if (RT_FAILURE(rc2))
10435 {
10436 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10437 rcStrict = rc2;
10438 }
10439 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10440 return rcStrict;
10441}
10442
10443
10444#ifndef HMVMX_USE_FUNCTION_TABLE
10445DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10446{
10447# ifdef DEBUG_ramshankar
10448# define RETURN_EXIT_CALL(a_CallExpr) \
10449 do { \
10450 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10451 VBOXSTRICTRC rcStrict = a_CallExpr; \
10452 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10453 return rcStrict; \
10454 } while (0)
10455# else
10456# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10457# endif
10458 switch (rcReason)
10459 {
10460 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10461 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10462 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10463 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10464 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10465 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10466 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10467 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10468 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10469 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10470 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10471 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10472 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10473 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10474 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10475 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10476 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10477 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10478 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10479 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10480 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10481 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10494
10495 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10496 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10497 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10498 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10499 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10500 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10501 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10502 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10503 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10504
10505 case VMX_EXIT_VMCLEAR:
10506 case VMX_EXIT_VMLAUNCH:
10507 case VMX_EXIT_VMPTRLD:
10508 case VMX_EXIT_VMPTRST:
10509 case VMX_EXIT_VMREAD:
10510 case VMX_EXIT_VMRESUME:
10511 case VMX_EXIT_VMWRITE:
10512 case VMX_EXIT_VMXOFF:
10513 case VMX_EXIT_VMXON:
10514 case VMX_EXIT_INVEPT:
10515 case VMX_EXIT_INVVPID:
10516 case VMX_EXIT_VMFUNC:
10517 case VMX_EXIT_XSAVES:
10518 case VMX_EXIT_XRSTORS:
10519 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10520 case VMX_EXIT_ENCLS:
10521 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10522 case VMX_EXIT_PML_FULL:
10523 default:
10524 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10525 }
10526#undef RETURN_EXIT_CALL
10527}
10528#endif /* !HMVMX_USE_FUNCTION_TABLE */
10529
10530
10531#ifdef VBOX_STRICT
10532/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10533# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10534 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10535
10536# define HMVMX_ASSERT_PREEMPT_CPUID() \
10537 do { \
10538 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10539 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10540 } while (0)
10541
10542# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10543 do { \
10544 AssertPtr(pVCpu); \
10545 AssertPtr(pMixedCtx); \
10546 AssertPtr(pVmxTransient); \
10547 Assert(pVmxTransient->fVMEntryFailed == false); \
10548 Assert(ASMIntAreEnabled()); \
10549 HMVMX_ASSERT_PREEMPT_SAFE(); \
10550 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10551 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10552 HMVMX_ASSERT_PREEMPT_SAFE(); \
10553 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10554 HMVMX_ASSERT_PREEMPT_CPUID(); \
10555 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10556 } while (0)
10557
10558# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10559 do { \
10560 Log4Func(("\n")); \
10561 } while (0)
10562#else /* nonstrict builds: */
10563# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10564 do { \
10565 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10566 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10567 } while (0)
10568# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10569#endif
10570
10571
10572/**
10573 * Advances the guest RIP by the specified number of bytes.
10574 *
10575 * @param pVCpu The cross context virtual CPU structure.
10576 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10577 * out-of-sync. Make sure to update the required fields
10578 * before using them.
10579 * @param cbInstr Number of bytes to advance the RIP by.
10580 *
10581 * @remarks No-long-jump zone!!!
10582 */
10583DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10584{
10585 /* Advance the RIP. */
10586 pMixedCtx->rip += cbInstr;
10587 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10588
10589 /* Update interrupt inhibition. */
10590 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10591 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10592 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10593}
10594
10595
10596/**
10597 * Advances the guest RIP after reading it from the VMCS.
10598 *
10599 * @returns VBox status code, no informational status codes.
10600 * @param pVCpu The cross context virtual CPU structure.
10601 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10602 * out-of-sync. Make sure to update the required fields
10603 * before using them.
10604 * @param pVmxTransient Pointer to the VMX transient structure.
10605 *
10606 * @remarks No-long-jump zone!!!
10607 */
10608static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10609{
10610 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10611 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10612 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10613 AssertRCReturn(rc, rc);
10614
10615 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10616
10617 /*
10618 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10619 * pending debug exception field as it takes care of priority of events.
10620 *
10621 * See Intel spec. 32.2.1 "Debug Exceptions".
10622 */
10623 if ( !pVCpu->hm.s.fSingleInstruction
10624 && pMixedCtx->eflags.Bits.u1TF)
10625 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10626
10627 return VINF_SUCCESS;
10628}
10629
10630
10631/**
10632 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10633 * and update error record fields accordingly.
10634 *
10635 * @return VMX_IGS_* return codes.
10636 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10637 * wrong with the guest state.
10638 *
10639 * @param pVM The cross context VM structure.
10640 * @param pVCpu The cross context virtual CPU structure.
10641 * @param pCtx Pointer to the guest-CPU state.
10642 *
10643 * @remarks This function assumes our cache of the VMCS controls
10644 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10645 */
10646static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10647{
10648#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10649#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10650 uError = (err); \
10651 break; \
10652 } else do { } while (0)
10653
10654 int rc;
10655 uint32_t uError = VMX_IGS_ERROR;
10656 uint32_t u32Val;
10657 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10658
10659 do
10660 {
10661 /*
10662 * CR0.
10663 */
10664 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10665 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10666 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10667 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10668 if (fUnrestrictedGuest)
10669 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10670
10671 uint32_t u32GuestCR0;
10672 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10673 AssertRCBreak(rc);
10674 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10675 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10676 if ( !fUnrestrictedGuest
10677 && (u32GuestCR0 & X86_CR0_PG)
10678 && !(u32GuestCR0 & X86_CR0_PE))
10679 {
10680 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10681 }
10682
10683 /*
10684 * CR4.
10685 */
10686 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10687 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10688
10689 uint32_t u32GuestCR4;
10690 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10691 AssertRCBreak(rc);
10692 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10693 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10694
10695 /*
10696 * IA32_DEBUGCTL MSR.
10697 */
10698 uint64_t u64Val;
10699 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10700 AssertRCBreak(rc);
10701 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10702 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10703 {
10704 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10705 }
10706 uint64_t u64DebugCtlMsr = u64Val;
10707
10708#ifdef VBOX_STRICT
10709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10710 AssertRCBreak(rc);
10711 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10712#endif
10713 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10714
10715 /*
10716 * RIP and RFLAGS.
10717 */
10718 uint32_t u32Eflags;
10719#if HC_ARCH_BITS == 64
10720 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10721 AssertRCBreak(rc);
10722 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10723 if ( !fLongModeGuest
10724 || !pCtx->cs.Attr.n.u1Long)
10725 {
10726 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10727 }
10728 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10729 * must be identical if the "IA-32e mode guest" VM-entry
10730 * control is 1 and CS.L is 1. No check applies if the
10731 * CPU supports 64 linear-address bits. */
10732
10733 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10734 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10735 AssertRCBreak(rc);
10736 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10737 VMX_IGS_RFLAGS_RESERVED);
10738 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10739 u32Eflags = u64Val;
10740#else
10741 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10742 AssertRCBreak(rc);
10743 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10744 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10745#endif
10746
10747 if ( fLongModeGuest
10748 || ( fUnrestrictedGuest
10749 && !(u32GuestCR0 & X86_CR0_PE)))
10750 {
10751 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10752 }
10753
10754 uint32_t u32EntryInfo;
10755 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10756 AssertRCBreak(rc);
10757 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10758 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10759 {
10760 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10761 }
10762
10763 /*
10764 * 64-bit checks.
10765 */
10766#if HC_ARCH_BITS == 64
10767 if (fLongModeGuest)
10768 {
10769 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10770 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10771 }
10772
10773 if ( !fLongModeGuest
10774 && (u32GuestCR4 & X86_CR4_PCIDE))
10775 {
10776 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10777 }
10778
10779 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10780 * 51:32 beyond the processor's physical-address width are 0. */
10781
10782 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10783 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10784 {
10785 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10786 }
10787
10788 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10789 AssertRCBreak(rc);
10790 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10791
10792 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10793 AssertRCBreak(rc);
10794 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10795#endif
10796
10797 /*
10798 * PERF_GLOBAL MSR.
10799 */
10800 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10801 {
10802 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10803 AssertRCBreak(rc);
10804 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10805 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10806 }
10807
10808 /*
10809 * PAT MSR.
10810 */
10811 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10812 {
10813 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10814 AssertRCBreak(rc);
10815 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10816 for (unsigned i = 0; i < 8; i++)
10817 {
10818 uint8_t u8Val = (u64Val & 0xff);
10819 if ( u8Val != 0 /* UC */
10820 && u8Val != 1 /* WC */
10821 && u8Val != 4 /* WT */
10822 && u8Val != 5 /* WP */
10823 && u8Val != 6 /* WB */
10824 && u8Val != 7 /* UC- */)
10825 {
10826 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10827 }
10828 u64Val >>= 8;
10829 }
10830 }
10831
10832 /*
10833 * EFER MSR.
10834 */
10835 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10836 {
10837 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10838 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10839 AssertRCBreak(rc);
10840 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10841 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10842 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10843 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10844 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10845 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10846 || !(u32GuestCR0 & X86_CR0_PG)
10847 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10848 VMX_IGS_EFER_LMA_LME_MISMATCH);
10849 }
10850
10851 /*
10852 * Segment registers.
10853 */
10854 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10855 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10856 if (!(u32Eflags & X86_EFL_VM))
10857 {
10858 /* CS */
10859 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10860 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10861 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10862 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10863 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10864 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10865 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10866 /* CS cannot be loaded with NULL in protected mode. */
10867 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10868 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10869 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10870 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10871 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10872 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10873 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10874 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10875 else
10876 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10877
10878 /* SS */
10879 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10880 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10881 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10882 if ( !(pCtx->cr0 & X86_CR0_PE)
10883 || pCtx->cs.Attr.n.u4Type == 3)
10884 {
10885 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10886 }
10887 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10888 {
10889 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10890 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10891 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10892 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10893 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10894 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10895 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10896 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10897 }
10898
10899 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10900 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10901 {
10902 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10903 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10904 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10905 || pCtx->ds.Attr.n.u4Type > 11
10906 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10907 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10908 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10909 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10910 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10911 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10912 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10913 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10914 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10915 }
10916 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10917 {
10918 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10919 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10920 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10921 || pCtx->es.Attr.n.u4Type > 11
10922 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10923 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10924 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10925 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10926 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10927 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10928 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10929 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10930 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10931 }
10932 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10933 {
10934 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10935 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10936 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10937 || pCtx->fs.Attr.n.u4Type > 11
10938 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10939 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10940 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10941 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10942 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10943 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10944 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10945 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10946 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10947 }
10948 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10949 {
10950 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10951 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10952 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10953 || pCtx->gs.Attr.n.u4Type > 11
10954 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10955 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10956 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10957 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10958 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10959 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10960 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10961 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10962 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10963 }
10964 /* 64-bit capable CPUs. */
10965#if HC_ARCH_BITS == 64
10966 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10967 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10968 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10969 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10970 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10971 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10972 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10973 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10974 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10975 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10976 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10977#endif
10978 }
10979 else
10980 {
10981 /* V86 mode checks. */
10982 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10983 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10984 {
10985 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10986 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10987 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10988 }
10989 else
10990 {
10991 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10992 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10993 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10994 }
10995
10996 /* CS */
10997 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10998 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10999 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11000 /* SS */
11001 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11002 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11003 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11004 /* DS */
11005 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11006 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11007 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11008 /* ES */
11009 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11010 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11011 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11012 /* FS */
11013 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11014 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11015 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11016 /* GS */
11017 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11018 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11019 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11020 /* 64-bit capable CPUs. */
11021#if HC_ARCH_BITS == 64
11022 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11023 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11024 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11025 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11026 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11027 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11028 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11029 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11030 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11031 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11032 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11033#endif
11034 }
11035
11036 /*
11037 * TR.
11038 */
11039 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11040 /* 64-bit capable CPUs. */
11041#if HC_ARCH_BITS == 64
11042 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11043#endif
11044 if (fLongModeGuest)
11045 {
11046 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11047 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11048 }
11049 else
11050 {
11051 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11052 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11053 VMX_IGS_TR_ATTR_TYPE_INVALID);
11054 }
11055 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11056 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11057 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11058 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11059 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11060 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11061 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11062 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11063
11064 /*
11065 * GDTR and IDTR.
11066 */
11067#if HC_ARCH_BITS == 64
11068 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11069 AssertRCBreak(rc);
11070 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11071
11072 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11073 AssertRCBreak(rc);
11074 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11075#endif
11076
11077 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11078 AssertRCBreak(rc);
11079 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11080
11081 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11082 AssertRCBreak(rc);
11083 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11084
11085 /*
11086 * Guest Non-Register State.
11087 */
11088 /* Activity State. */
11089 uint32_t u32ActivityState;
11090 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11091 AssertRCBreak(rc);
11092 HMVMX_CHECK_BREAK( !u32ActivityState
11093 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11094 VMX_IGS_ACTIVITY_STATE_INVALID);
11095 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11096 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11097 uint32_t u32IntrState;
11098 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11099 AssertRCBreak(rc);
11100 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11101 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11102 {
11103 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11104 }
11105
11106 /** @todo Activity state and injecting interrupts. Left as a todo since we
11107 * currently don't use activity states but ACTIVE. */
11108
11109 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11110 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11111
11112 /* Guest interruptibility-state. */
11113 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11114 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11115 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11116 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11117 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11118 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11119 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11120 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11121 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11122 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11123 {
11124 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11125 {
11126 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11127 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11128 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11129 }
11130 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11131 {
11132 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11133 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11134 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11135 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11136 }
11137 }
11138 /** @todo Assumes the processor is not in SMM. */
11139 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11140 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11141 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11142 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11143 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11144 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11145 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11146 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11147 {
11148 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11149 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11150 }
11151
11152 /* Pending debug exceptions. */
11153#if HC_ARCH_BITS == 64
11154 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11155 AssertRCBreak(rc);
11156 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11157 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11158 u32Val = u64Val; /* For pending debug exceptions checks below. */
11159#else
11160 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11161 AssertRCBreak(rc);
11162 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11163 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11164#endif
11165
11166 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11167 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11168 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11169 {
11170 if ( (u32Eflags & X86_EFL_TF)
11171 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11172 {
11173 /* Bit 14 is PendingDebug.BS. */
11174 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11175 }
11176 if ( !(u32Eflags & X86_EFL_TF)
11177 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11178 {
11179 /* Bit 14 is PendingDebug.BS. */
11180 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11181 }
11182 }
11183
11184 /* VMCS link pointer. */
11185 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11186 AssertRCBreak(rc);
11187 if (u64Val != UINT64_C(0xffffffffffffffff))
11188 {
11189 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11190 /** @todo Bits beyond the processor's physical-address width MBZ. */
11191 /** @todo 32-bit located in memory referenced by value of this field (as a
11192 * physical address) must contain the processor's VMCS revision ID. */
11193 /** @todo SMM checks. */
11194 }
11195
11196 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11197 * not using Nested Paging? */
11198 if ( pVM->hm.s.fNestedPaging
11199 && !fLongModeGuest
11200 && CPUMIsGuestInPAEModeEx(pCtx))
11201 {
11202 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11203 AssertRCBreak(rc);
11204 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11205
11206 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11207 AssertRCBreak(rc);
11208 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11209
11210 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11211 AssertRCBreak(rc);
11212 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11213
11214 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11215 AssertRCBreak(rc);
11216 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11217 }
11218
11219 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11220 if (uError == VMX_IGS_ERROR)
11221 uError = VMX_IGS_REASON_NOT_FOUND;
11222 } while (0);
11223
11224 pVCpu->hm.s.u32HMError = uError;
11225 return uError;
11226
11227#undef HMVMX_ERROR_BREAK
11228#undef HMVMX_CHECK_BREAK
11229}
11230
11231/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11232/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11233/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11234
11235/** @name VM-exit handlers.
11236 * @{
11237 */
11238
11239/**
11240 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11241 */
11242HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11243{
11244 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11245 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11246 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11247 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11248 return VINF_SUCCESS;
11249 return VINF_EM_RAW_INTERRUPT;
11250}
11251
11252
11253/**
11254 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11255 */
11256HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257{
11258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11260
11261 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11262 AssertRCReturn(rc, rc);
11263
11264 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11265 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11266 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11267 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11268
11269 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11270 {
11271 /*
11272 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11273 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11274 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11275 *
11276 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11277 */
11278 VMXDispatchHostNmi();
11279 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11280 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11281 return VINF_SUCCESS;
11282 }
11283
11284 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11285 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11286 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11287 { /* likely */ }
11288 else
11289 {
11290 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11291 rcStrictRc1 = VINF_SUCCESS;
11292 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11293 return rcStrictRc1;
11294 }
11295
11296 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11297 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11298 switch (uIntType)
11299 {
11300 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11301 Assert(uVector == X86_XCPT_DB);
11302 /* no break */
11303 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11304 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11305 /* no break */
11306 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11307 {
11308 /*
11309 * If there's any exception caused as a result of event injection, go back to
11310 * the interpreter. The page-fault case is complicated and we manually handle
11311 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11312 * handled in hmR0VmxCheckExitDueToEventDelivery.
11313 */
11314 if (!pVCpu->hm.s.Event.fPending)
11315 { /* likely */ }
11316 else if ( uVector != X86_XCPT_PF
11317 && uVector != X86_XCPT_AC)
11318 {
11319 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11320 rc = VERR_EM_INTERPRETER;
11321 break;
11322 }
11323
11324 switch (uVector)
11325 {
11326 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11327 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11328 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11329 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11330 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11331 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11332 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11333
11334 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11335 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11336 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11337 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11338 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11339 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11340 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11341 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11342 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11343 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11344 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11345 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11346 default:
11347 {
11348 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11349 AssertRCReturn(rc, rc);
11350
11351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11352 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11353 {
11354 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11355 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11356 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11357
11358 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11359 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11360 AssertRCReturn(rc, rc);
11361 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11362 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11363 0 /* GCPtrFaultAddress */);
11364 AssertRCReturn(rc, rc);
11365 }
11366 else
11367 {
11368 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11369 pVCpu->hm.s.u32HMError = uVector;
11370 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11371 }
11372 break;
11373 }
11374 }
11375 break;
11376 }
11377
11378 default:
11379 {
11380 pVCpu->hm.s.u32HMError = uExitIntInfo;
11381 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11382 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11383 break;
11384 }
11385 }
11386 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11387 return rc;
11388}
11389
11390
11391/**
11392 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11393 */
11394HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11395{
11396 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11397
11398 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11399 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11400
11401 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11402 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11403 return VINF_SUCCESS;
11404}
11405
11406
11407/**
11408 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11409 */
11410HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11411{
11412 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11413 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11414 {
11415 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11416 HMVMX_RETURN_UNEXPECTED_EXIT();
11417 }
11418
11419 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11420
11421 /*
11422 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11423 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11424 */
11425 uint32_t uIntrState = 0;
11426 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11427 AssertRCReturn(rc, rc);
11428
11429 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11430 if ( fBlockSti
11431 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11432 {
11433 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11434 }
11435
11436 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11437 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11438
11439 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11440 return VINF_SUCCESS;
11441}
11442
11443
11444/**
11445 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11446 */
11447HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11448{
11449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11450 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11451 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11452}
11453
11454
11455/**
11456 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11457 */
11458HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11459{
11460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11462 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11463}
11464
11465
11466/**
11467 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11468 */
11469HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11470{
11471 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11472 PVM pVM = pVCpu->CTX_SUFF(pVM);
11473 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11474 if (RT_LIKELY(rc == VINF_SUCCESS))
11475 {
11476 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11477 Assert(pVmxTransient->cbInstr == 2);
11478 }
11479 else
11480 {
11481 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11482 rc = VERR_EM_INTERPRETER;
11483 }
11484 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11485 return rc;
11486}
11487
11488
11489/**
11490 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11491 */
11492HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11493{
11494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11495 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11496 AssertRCReturn(rc, rc);
11497
11498 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11499 return VINF_EM_RAW_EMULATE_INSTR;
11500
11501 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11502 HMVMX_RETURN_UNEXPECTED_EXIT();
11503}
11504
11505
11506/**
11507 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11508 */
11509HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11510{
11511 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11512 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11513 AssertRCReturn(rc, rc);
11514
11515 PVM pVM = pVCpu->CTX_SUFF(pVM);
11516 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11517 if (RT_LIKELY(rc == VINF_SUCCESS))
11518 {
11519 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11520 Assert(pVmxTransient->cbInstr == 2);
11521 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11522 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11523 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11524 }
11525 else
11526 rc = VERR_EM_INTERPRETER;
11527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11528 return rc;
11529}
11530
11531
11532/**
11533 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11534 */
11535HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11536{
11537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11538 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11539 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11540 AssertRCReturn(rc, rc);
11541
11542 PVM pVM = pVCpu->CTX_SUFF(pVM);
11543 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11544 if (RT_SUCCESS(rc))
11545 {
11546 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11547 Assert(pVmxTransient->cbInstr == 3);
11548 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11550 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11551 }
11552 else
11553 {
11554 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11555 rc = VERR_EM_INTERPRETER;
11556 }
11557 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11558 return rc;
11559}
11560
11561
11562/**
11563 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11564 */
11565HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11566{
11567 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11568 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11569 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11570 AssertRCReturn(rc, rc);
11571
11572 PVM pVM = pVCpu->CTX_SUFF(pVM);
11573 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11574 if (RT_LIKELY(rc == VINF_SUCCESS))
11575 {
11576 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11577 Assert(pVmxTransient->cbInstr == 2);
11578 }
11579 else
11580 {
11581 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11582 rc = VERR_EM_INTERPRETER;
11583 }
11584 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11585 return rc;
11586}
11587
11588
11589/**
11590 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11591 */
11592HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11596
11597 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11598 if (pVCpu->hm.s.fHypercallsEnabled)
11599 {
11600#if 0
11601 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11602#else
11603 /* Aggressive state sync. for now. */
11604 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11605 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11606 AssertRCReturn(rc, rc);
11607#endif
11608
11609 /* Perform the hypercall. */
11610 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11611 if (rcStrict == VINF_SUCCESS)
11612 {
11613 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11614 AssertRCReturn(rc, rc);
11615 }
11616 else
11617 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11618 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11619 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11620
11621 /* If the hypercall changes anything other than guest's general-purpose registers,
11622 we would need to reload the guest changed bits here before VM-entry. */
11623 }
11624 else
11625 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11626
11627 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11628 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11629 {
11630 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11631 rcStrict = VINF_SUCCESS;
11632 }
11633
11634 return rcStrict;
11635}
11636
11637
11638/**
11639 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11640 */
11641HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11642{
11643 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11644 PVM pVM = pVCpu->CTX_SUFF(pVM);
11645 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11646
11647 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11648 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11649 AssertRCReturn(rc, rc);
11650
11651 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11652 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11653 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11654 else
11655 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11656 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11658 return rcStrict;
11659}
11660
11661
11662/**
11663 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11664 */
11665HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11666{
11667 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11668 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11669 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11670 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11671 AssertRCReturn(rc, rc);
11672
11673 PVM pVM = pVCpu->CTX_SUFF(pVM);
11674 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11675 if (RT_LIKELY(rc == VINF_SUCCESS))
11676 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11677 else
11678 {
11679 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11680 rc = VERR_EM_INTERPRETER;
11681 }
11682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11683 return rc;
11684}
11685
11686
11687/**
11688 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11689 */
11690HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11691{
11692 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11693 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11694 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11695 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11696 AssertRCReturn(rc, rc);
11697
11698 PVM pVM = pVCpu->CTX_SUFF(pVM);
11699 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11700 rc = VBOXSTRICTRC_VAL(rc2);
11701 if (RT_LIKELY( rc == VINF_SUCCESS
11702 || rc == VINF_EM_HALT))
11703 {
11704 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11705 AssertRCReturn(rc3, rc3);
11706
11707 if ( rc == VINF_EM_HALT
11708 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11709 {
11710 rc = VINF_SUCCESS;
11711 }
11712 }
11713 else
11714 {
11715 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11716 rc = VERR_EM_INTERPRETER;
11717 }
11718 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11719 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11721 return rc;
11722}
11723
11724
11725/**
11726 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11727 */
11728HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11729{
11730 /*
11731 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11732 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11733 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11734 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11735 */
11736 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11737 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11738 HMVMX_RETURN_UNEXPECTED_EXIT();
11739}
11740
11741
11742/**
11743 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11744 */
11745HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11746{
11747 /*
11748 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11749 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11750 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11751 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11752 */
11753 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11754 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11755 HMVMX_RETURN_UNEXPECTED_EXIT();
11756}
11757
11758
11759/**
11760 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11761 */
11762HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11763{
11764 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11766 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11767 HMVMX_RETURN_UNEXPECTED_EXIT();
11768}
11769
11770
11771/**
11772 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11773 */
11774HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11775{
11776 /*
11777 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11778 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11779 * See Intel spec. 25.3 "Other Causes of VM-exits".
11780 */
11781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11782 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11783 HMVMX_RETURN_UNEXPECTED_EXIT();
11784}
11785
11786
11787/**
11788 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11789 * VM-exit.
11790 */
11791HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11792{
11793 /*
11794 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11795 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11796 *
11797 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11798 * See Intel spec. "23.8 Restrictions on VMX operation".
11799 */
11800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11801 return VINF_SUCCESS;
11802}
11803
11804
11805/**
11806 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11807 * VM-exit.
11808 */
11809HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11810{
11811 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11812 return VINF_EM_RESET;
11813}
11814
11815
11816/**
11817 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11818 */
11819HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11820{
11821 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11822 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11823
11824 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11825 AssertRCReturn(rc, rc);
11826
11827 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11828 rc = VINF_SUCCESS;
11829 else
11830 rc = VINF_EM_HALT;
11831
11832 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11833 if (rc != VINF_SUCCESS)
11834 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11835 return rc;
11836}
11837
11838
11839/**
11840 * VM-exit handler for instructions that result in a \#UD exception delivered to
11841 * the guest.
11842 */
11843HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11844{
11845 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11846 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11847 return VINF_SUCCESS;
11848}
11849
11850
11851/**
11852 * VM-exit handler for expiry of the VMX preemption timer.
11853 */
11854HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855{
11856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857
11858 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11859 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11860
11861 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11862 PVM pVM = pVCpu->CTX_SUFF(pVM);
11863 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11865 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11866}
11867
11868
11869/**
11870 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11871 */
11872HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11873{
11874 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11875
11876 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11877 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11878 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11879 AssertRCReturn(rc, rc);
11880
11881 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11882 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11883
11884 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11885
11886 return rcStrict;
11887}
11888
11889
11890/**
11891 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11892 */
11893HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11894{
11895 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11896
11897 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11898 /** @todo implement EMInterpretInvpcid() */
11899 return VERR_EM_INTERPRETER;
11900}
11901
11902
11903/**
11904 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11905 * Error VM-exit.
11906 */
11907HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11908{
11909 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11910 AssertRCReturn(rc, rc);
11911
11912 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11913 AssertRCReturn(rc, rc);
11914
11915 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11916 NOREF(uInvalidReason);
11917
11918#ifdef VBOX_STRICT
11919 uint32_t uIntrState;
11920 RTHCUINTREG uHCReg;
11921 uint64_t u64Val;
11922 uint32_t u32Val;
11923
11924 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11925 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11926 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11927 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11928 AssertRCReturn(rc, rc);
11929
11930 Log4(("uInvalidReason %u\n", uInvalidReason));
11931 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11932 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11933 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11934 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11935
11936 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11937 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11938 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11939 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11940 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11941 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11942 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11943 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11944 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11945 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11946 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11947 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11948#else
11949 NOREF(pVmxTransient);
11950#endif
11951
11952 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11953 return VERR_VMX_INVALID_GUEST_STATE;
11954}
11955
11956
11957/**
11958 * VM-exit handler for VM-entry failure due to an MSR-load
11959 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11960 */
11961HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11962{
11963 NOREF(pVmxTransient);
11964 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11965 HMVMX_RETURN_UNEXPECTED_EXIT();
11966}
11967
11968
11969/**
11970 * VM-exit handler for VM-entry failure due to a machine-check event
11971 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11972 */
11973HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11974{
11975 NOREF(pVmxTransient);
11976 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11977 HMVMX_RETURN_UNEXPECTED_EXIT();
11978}
11979
11980
11981/**
11982 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11983 * theory.
11984 */
11985HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11986{
11987 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11988 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11989 return VERR_VMX_UNDEFINED_EXIT_CODE;
11990}
11991
11992
11993/**
11994 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11995 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11996 * Conditional VM-exit.
11997 */
11998HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11999{
12000 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12001
12002 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12004 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12005 return VERR_EM_INTERPRETER;
12006 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12007 HMVMX_RETURN_UNEXPECTED_EXIT();
12008}
12009
12010
12011/**
12012 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12013 */
12014HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12015{
12016 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12017
12018 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12019 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12020 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12021 return VERR_EM_INTERPRETER;
12022 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12023 HMVMX_RETURN_UNEXPECTED_EXIT();
12024}
12025
12026
12027/**
12028 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12029 */
12030HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12031{
12032 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12033
12034 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12035 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12036 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12037 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12038 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12039 {
12040 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12041 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12042 }
12043 AssertRCReturn(rc, rc);
12044 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12045
12046#ifdef VBOX_STRICT
12047 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12048 {
12049 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12050 && pMixedCtx->ecx != MSR_K6_EFER)
12051 {
12052 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12053 pMixedCtx->ecx));
12054 HMVMX_RETURN_UNEXPECTED_EXIT();
12055 }
12056 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12057 {
12058 VMXMSREXITREAD enmRead;
12059 VMXMSREXITWRITE enmWrite;
12060 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12061 AssertRCReturn(rc2, rc2);
12062 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12063 {
12064 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12065 HMVMX_RETURN_UNEXPECTED_EXIT();
12066 }
12067 }
12068 }
12069#endif
12070
12071 PVM pVM = pVCpu->CTX_SUFF(pVM);
12072 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12073 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12074 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12076 if (RT_SUCCESS(rc))
12077 {
12078 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12079 Assert(pVmxTransient->cbInstr == 2);
12080 }
12081 return rc;
12082}
12083
12084
12085/**
12086 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12087 */
12088HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12089{
12090 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12091 PVM pVM = pVCpu->CTX_SUFF(pVM);
12092 int rc = VINF_SUCCESS;
12093
12094 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12095 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12096 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12097 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12098 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12099 {
12100 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12101 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12102 }
12103 AssertRCReturn(rc, rc);
12104 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12105
12106 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12107 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12108 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12109
12110 if (RT_SUCCESS(rc))
12111 {
12112 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12113
12114 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12115 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12116 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12117 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12118 {
12119 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12120 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12121 EMInterpretWrmsr() changes it. */
12122 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12123 }
12124 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12125 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12126 else if (pMixedCtx->ecx == MSR_K6_EFER)
12127 {
12128 /*
12129 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12130 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12131 * the other bits as well, SCE and NXE. See @bugref{7368}.
12132 */
12133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12134 }
12135
12136 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12137 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12138 {
12139 switch (pMixedCtx->ecx)
12140 {
12141 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12142 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12143 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12144 case MSR_K8_FS_BASE: /* no break */
12145 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12146 case MSR_K6_EFER: /* already handled above */ break;
12147 default:
12148 {
12149 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12150 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12151 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12152 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12153 break;
12154 }
12155 }
12156 }
12157#ifdef VBOX_STRICT
12158 else
12159 {
12160 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12161 switch (pMixedCtx->ecx)
12162 {
12163 case MSR_IA32_SYSENTER_CS:
12164 case MSR_IA32_SYSENTER_EIP:
12165 case MSR_IA32_SYSENTER_ESP:
12166 case MSR_K8_FS_BASE:
12167 case MSR_K8_GS_BASE:
12168 {
12169 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12170 HMVMX_RETURN_UNEXPECTED_EXIT();
12171 }
12172
12173 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12174 default:
12175 {
12176 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12177 {
12178 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12179 if (pMixedCtx->ecx != MSR_K6_EFER)
12180 {
12181 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12182 pMixedCtx->ecx));
12183 HMVMX_RETURN_UNEXPECTED_EXIT();
12184 }
12185 }
12186
12187 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12188 {
12189 VMXMSREXITREAD enmRead;
12190 VMXMSREXITWRITE enmWrite;
12191 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12192 AssertRCReturn(rc2, rc2);
12193 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12194 {
12195 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12196 HMVMX_RETURN_UNEXPECTED_EXIT();
12197 }
12198 }
12199 break;
12200 }
12201 }
12202 }
12203#endif /* VBOX_STRICT */
12204 }
12205 return rc;
12206}
12207
12208
12209/**
12210 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12211 */
12212HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12213{
12214 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12215
12216 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12217 return VINF_EM_RAW_INTERRUPT;
12218}
12219
12220
12221/**
12222 * VM-exit handler for when the TPR value is lowered below the specified
12223 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12224 */
12225HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12226{
12227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12228 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12229
12230 /*
12231 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12232 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12233 */
12234 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12235 return VINF_SUCCESS;
12236}
12237
12238
12239/**
12240 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12241 * VM-exit.
12242 *
12243 * @retval VINF_SUCCESS when guest execution can continue.
12244 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12245 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12246 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12247 * interpreter.
12248 */
12249HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12250{
12251 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12252 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12253 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12254 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12255 AssertRCReturn(rc, rc);
12256
12257 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12258 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12259 PVM pVM = pVCpu->CTX_SUFF(pVM);
12260 VBOXSTRICTRC rcStrict;
12261 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12262 switch (uAccessType)
12263 {
12264 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12265 {
12266 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12267 AssertRCReturn(rc, rc);
12268
12269 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12270 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12271 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12272 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12273 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12274 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12275 {
12276 case 0: /* CR0 */
12277 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12278 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12279 break;
12280 case 2: /* CR2 */
12281 /* Nothing to do here, CR2 it's not part of the VMCS. */
12282 break;
12283 case 3: /* CR3 */
12284 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12285 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12286 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12287 break;
12288 case 4: /* CR4 */
12289 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12290 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12291 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12292 break;
12293 case 8: /* CR8 */
12294 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12295 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12296 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12297 break;
12298 default:
12299 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12300 break;
12301 }
12302
12303 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12304 break;
12305 }
12306
12307 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12308 {
12309 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12310 AssertRCReturn(rc, rc);
12311
12312 Assert( !pVM->hm.s.fNestedPaging
12313 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12314 || pVCpu->hm.s.fUsingDebugLoop
12315 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12316
12317 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12318 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12319 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12320
12321 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12322 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12323 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12324 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12326 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12327 VBOXSTRICTRC_VAL(rcStrict)));
12328 break;
12329 }
12330
12331 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12332 {
12333 AssertRCReturn(rc, rc);
12334 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12335 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12336 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12337 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12338 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12339 break;
12340 }
12341
12342 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12343 {
12344 AssertRCReturn(rc, rc);
12345 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12346 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12347 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12348 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12349 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12350 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12351 break;
12352 }
12353
12354 default:
12355 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12356 VERR_VMX_UNEXPECTED_EXCEPTION);
12357 }
12358
12359 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12360 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12361 NOREF(pVM);
12362 return rcStrict;
12363}
12364
12365
12366/**
12367 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12368 * VM-exit.
12369 */
12370HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12371{
12372 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12373 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12374
12375 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12376 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12377 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12378 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12379 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12380 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12381 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12382 AssertRCReturn(rc2, rc2);
12383
12384 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12385 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12386 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12387 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12388 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12389 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12390 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12391 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12392 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12393
12394 /* I/O operation lookup arrays. */
12395 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12396 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12397
12398 VBOXSTRICTRC rcStrict;
12399 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12400 uint32_t const cbInstr = pVmxTransient->cbInstr;
12401 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12402 PVM pVM = pVCpu->CTX_SUFF(pVM);
12403 if (fIOString)
12404 {
12405#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12406 See @bugref{5752#c158}. Should work now. */
12407 /*
12408 * INS/OUTS - I/O String instruction.
12409 *
12410 * Use instruction-information if available, otherwise fall back on
12411 * interpreting the instruction.
12412 */
12413 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12414 fIOWrite ? 'w' : 'r'));
12415 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12416 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12417 {
12418 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12419 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12420 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12421 AssertRCReturn(rc2, rc2);
12422 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12423 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12424 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12425 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12426 if (fIOWrite)
12427 {
12428 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12429 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12430 }
12431 else
12432 {
12433 /*
12434 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12435 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12436 * See Intel Instruction spec. for "INS".
12437 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12438 */
12439 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12440 }
12441 }
12442 else
12443 {
12444 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12445 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12446 AssertRCReturn(rc2, rc2);
12447 rcStrict = IEMExecOne(pVCpu);
12448 }
12449 /** @todo IEM needs to be setting these flags somehow. */
12450 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12451 fUpdateRipAlready = true;
12452#else
12453 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12454 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12455 if (RT_SUCCESS(rcStrict))
12456 {
12457 if (fIOWrite)
12458 {
12459 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12460 (DISCPUMODE)pDis->uAddrMode, cbValue);
12461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12462 }
12463 else
12464 {
12465 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12466 (DISCPUMODE)pDis->uAddrMode, cbValue);
12467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12468 }
12469 }
12470 else
12471 {
12472 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12473 pMixedCtx->rip));
12474 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12475 }
12476#endif
12477 }
12478 else
12479 {
12480 /*
12481 * IN/OUT - I/O instruction.
12482 */
12483 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12484 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12485 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12486 if (fIOWrite)
12487 {
12488 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12490 }
12491 else
12492 {
12493 uint32_t u32Result = 0;
12494 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12495 if (IOM_SUCCESS(rcStrict))
12496 {
12497 /* Save result of I/O IN instr. in AL/AX/EAX. */
12498 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12499 }
12500 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12501 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12503 }
12504 }
12505
12506 if (IOM_SUCCESS(rcStrict))
12507 {
12508 if (!fUpdateRipAlready)
12509 {
12510 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12511 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12512 }
12513
12514 /*
12515 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12516 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12517 */
12518 if (fIOString)
12519 {
12520 /** @todo Single-step for INS/OUTS with REP prefix? */
12521 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12522 }
12523 else if ( !fDbgStepping
12524 && fGstStepping)
12525 {
12526 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12527 }
12528
12529 /*
12530 * If any I/O breakpoints are armed, we need to check if one triggered
12531 * and take appropriate action.
12532 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12533 */
12534 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12535 AssertRCReturn(rc2, rc2);
12536
12537 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12538 * execution engines about whether hyper BPs and such are pending. */
12539 uint32_t const uDr7 = pMixedCtx->dr[7];
12540 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12541 && X86_DR7_ANY_RW_IO(uDr7)
12542 && (pMixedCtx->cr4 & X86_CR4_DE))
12543 || DBGFBpIsHwIoArmed(pVM)))
12544 {
12545 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12546
12547 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12548 VMMRZCallRing3Disable(pVCpu);
12549 HM_DISABLE_PREEMPT();
12550
12551 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12552
12553 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12554 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12555 {
12556 /* Raise #DB. */
12557 if (fIsGuestDbgActive)
12558 ASMSetDR6(pMixedCtx->dr[6]);
12559 if (pMixedCtx->dr[7] != uDr7)
12560 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12561
12562 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12563 }
12564 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12565 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12566 else if ( rcStrict2 != VINF_SUCCESS
12567 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12568 rcStrict = rcStrict2;
12569 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12570
12571 HM_RESTORE_PREEMPT();
12572 VMMRZCallRing3Enable(pVCpu);
12573 }
12574 }
12575
12576#ifdef VBOX_STRICT
12577 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12578 Assert(!fIOWrite);
12579 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12580 Assert(fIOWrite);
12581 else
12582 {
12583#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12584 * statuses, that the VMM device and some others may return. See
12585 * IOM_SUCCESS() for guidance. */
12586 AssertMsg( RT_FAILURE(rcStrict)
12587 || rcStrict == VINF_SUCCESS
12588 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12589 || rcStrict == VINF_EM_DBG_BREAKPOINT
12590 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12591 || rcStrict == VINF_EM_RAW_TO_R3
12592 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12593#endif
12594 }
12595#endif
12596
12597 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12598 return rcStrict;
12599}
12600
12601
12602/**
12603 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12604 * VM-exit.
12605 */
12606HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12607{
12608 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12609
12610 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12611 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12612 AssertRCReturn(rc, rc);
12613 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12614 {
12615 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12616 AssertRCReturn(rc, rc);
12617 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12618 {
12619 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12620
12621 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12622 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12623
12624 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12625 Assert(!pVCpu->hm.s.Event.fPending);
12626 pVCpu->hm.s.Event.fPending = true;
12627 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12628 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12629 AssertRCReturn(rc, rc);
12630 if (fErrorCodeValid)
12631 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12632 else
12633 pVCpu->hm.s.Event.u32ErrCode = 0;
12634 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12635 && uVector == X86_XCPT_PF)
12636 {
12637 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12638 }
12639
12640 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12642 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12643 }
12644 }
12645
12646 /* Fall back to the interpreter to emulate the task-switch. */
12647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12648 return VERR_EM_INTERPRETER;
12649}
12650
12651
12652/**
12653 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12654 */
12655HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12656{
12657 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12658 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12659 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12660 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12661 AssertRCReturn(rc, rc);
12662 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12663 return VINF_EM_DBG_STEPPED;
12664}
12665
12666
12667/**
12668 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12669 */
12670HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12671{
12672 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12673
12674 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12675
12676 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12677 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12678 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12679 {
12680 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12681 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12682 {
12683 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12684 return VERR_EM_INTERPRETER;
12685 }
12686 }
12687 else
12688 {
12689 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12690 rcStrict1 = VINF_SUCCESS;
12691 return rcStrict1;
12692 }
12693
12694#if 0
12695 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12696 * just sync the whole thing. */
12697 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12698#else
12699 /* Aggressive state sync. for now. */
12700 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12701 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12702 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12703#endif
12704 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12705 AssertRCReturn(rc, rc);
12706
12707 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12708 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12709 VBOXSTRICTRC rcStrict2;
12710 switch (uAccessType)
12711 {
12712 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12713 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12714 {
12715 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12716 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12717 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12718
12719 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12720 GCPhys &= PAGE_BASE_GC_MASK;
12721 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12722 PVM pVM = pVCpu->CTX_SUFF(pVM);
12723 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12724 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12725
12726 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12727 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12728 CPUMCTX2CORE(pMixedCtx), GCPhys);
12729 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12730 if ( rcStrict2 == VINF_SUCCESS
12731 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12732 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12733 {
12734 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12735 | HM_CHANGED_GUEST_RSP
12736 | HM_CHANGED_GUEST_RFLAGS
12737 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12738 rcStrict2 = VINF_SUCCESS;
12739 }
12740 break;
12741 }
12742
12743 default:
12744 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12745 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12746 break;
12747 }
12748
12749 if (rcStrict2 != VINF_SUCCESS)
12750 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12751 return rcStrict2;
12752}
12753
12754
12755/**
12756 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12757 * VM-exit.
12758 */
12759HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12760{
12761 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12762
12763 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12764 if (pVmxTransient->fWasGuestDebugStateActive)
12765 {
12766 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12767 HMVMX_RETURN_UNEXPECTED_EXIT();
12768 }
12769
12770 if ( !pVCpu->hm.s.fSingleInstruction
12771 && !pVmxTransient->fWasHyperDebugStateActive)
12772 {
12773 Assert(!DBGFIsStepping(pVCpu));
12774 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12775
12776 /* Don't intercept MOV DRx any more. */
12777 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12778 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12779 AssertRCReturn(rc, rc);
12780
12781 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12782 VMMRZCallRing3Disable(pVCpu);
12783 HM_DISABLE_PREEMPT();
12784
12785 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12786 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12787 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12788
12789 HM_RESTORE_PREEMPT();
12790 VMMRZCallRing3Enable(pVCpu);
12791
12792#ifdef VBOX_WITH_STATISTICS
12793 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12794 AssertRCReturn(rc, rc);
12795 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12797 else
12798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12799#endif
12800 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12801 return VINF_SUCCESS;
12802 }
12803
12804 /*
12805 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12806 * Update the segment registers and DR7 from the CPU.
12807 */
12808 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12809 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12810 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12811 AssertRCReturn(rc, rc);
12812 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12813
12814 PVM pVM = pVCpu->CTX_SUFF(pVM);
12815 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12816 {
12817 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12818 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12819 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12820 if (RT_SUCCESS(rc))
12821 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12823 }
12824 else
12825 {
12826 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12827 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12828 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12829 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12830 }
12831
12832 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12833 if (RT_SUCCESS(rc))
12834 {
12835 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12836 AssertRCReturn(rc2, rc2);
12837 return VINF_SUCCESS;
12838 }
12839 return rc;
12840}
12841
12842
12843/**
12844 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12845 * Conditional VM-exit.
12846 */
12847HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12848{
12849 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12850 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12851
12852 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12853 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12854 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12855 {
12856 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12857 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12858 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12859 {
12860 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12861 return VERR_EM_INTERPRETER;
12862 }
12863 }
12864 else
12865 {
12866 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12867 rcStrict1 = VINF_SUCCESS;
12868 return rcStrict1;
12869 }
12870
12871 RTGCPHYS GCPhys = 0;
12872 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12873
12874#if 0
12875 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12876#else
12877 /* Aggressive state sync. for now. */
12878 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12879 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12880 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12881#endif
12882 AssertRCReturn(rc, rc);
12883
12884 /*
12885 * If we succeed, resume guest execution.
12886 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12887 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12888 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12889 * weird case. See @bugref{6043}.
12890 */
12891 PVM pVM = pVCpu->CTX_SUFF(pVM);
12892 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12893 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12894 if ( rcStrict2 == VINF_SUCCESS
12895 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12896 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12897 {
12898 /* Successfully handled MMIO operation. */
12899 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12900 | HM_CHANGED_GUEST_RSP
12901 | HM_CHANGED_GUEST_RFLAGS
12902 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12903 return VINF_SUCCESS;
12904 }
12905 return rcStrict2;
12906}
12907
12908
12909/**
12910 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12911 * VM-exit.
12912 */
12913HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12914{
12915 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12916 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12917
12918 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12919 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12920 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12921 {
12922 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12923 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12924 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12925 }
12926 else
12927 {
12928 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12929 rcStrict1 = VINF_SUCCESS;
12930 return rcStrict1;
12931 }
12932
12933 RTGCPHYS GCPhys = 0;
12934 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12935 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12936#if 0
12937 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12938#else
12939 /* Aggressive state sync. for now. */
12940 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12941 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12942 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12943#endif
12944 AssertRCReturn(rc, rc);
12945
12946 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12947 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12948
12949 RTGCUINT uErrorCode = 0;
12950 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12951 uErrorCode |= X86_TRAP_PF_ID;
12952 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12953 uErrorCode |= X86_TRAP_PF_RW;
12954 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12955 uErrorCode |= X86_TRAP_PF_P;
12956
12957 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12958
12959 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12960 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12961
12962 /* Handle the pagefault trap for the nested shadow table. */
12963 PVM pVM = pVCpu->CTX_SUFF(pVM);
12964 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12965 TRPMResetTrap(pVCpu);
12966
12967 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12968 if ( rcStrict2 == VINF_SUCCESS
12969 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12970 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12971 {
12972 /* Successfully synced our nested page tables. */
12973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12974 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12975 | HM_CHANGED_GUEST_RSP
12976 | HM_CHANGED_GUEST_RFLAGS);
12977 return VINF_SUCCESS;
12978 }
12979
12980 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12981 return rcStrict2;
12982}
12983
12984/** @} */
12985
12986/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12987/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12988/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12989
12990/** @name VM-exit exception handlers.
12991 * @{
12992 */
12993
12994/**
12995 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12996 */
12997static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12998{
12999 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13001
13002 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13003 AssertRCReturn(rc, rc);
13004
13005 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13006 {
13007 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13008 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13009
13010 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13011 * provides VM-exit instruction length. If this causes problem later,
13012 * disassemble the instruction like it's done on AMD-V. */
13013 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13014 AssertRCReturn(rc2, rc2);
13015 return rc;
13016 }
13017
13018 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13019 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13020 return rc;
13021}
13022
13023
13024/**
13025 * VM-exit exception handler for \#BP (Breakpoint exception).
13026 */
13027static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13028{
13029 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13031
13032 /** @todo Try optimize this by not saving the entire guest state unless
13033 * really needed. */
13034 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13035 AssertRCReturn(rc, rc);
13036
13037 PVM pVM = pVCpu->CTX_SUFF(pVM);
13038 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13039 if (rc == VINF_EM_RAW_GUEST_TRAP)
13040 {
13041 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13042 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13043 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13044 AssertRCReturn(rc, rc);
13045
13046 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13047 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13048 }
13049
13050 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13051 return rc;
13052}
13053
13054
13055/**
13056 * VM-exit exception handler for \#AC (alignment check exception).
13057 */
13058static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13059{
13060 RT_NOREF_PV(pMixedCtx);
13061 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13062
13063 /*
13064 * Re-inject it. We'll detect any nesting before getting here.
13065 */
13066 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13067 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13068 AssertRCReturn(rc, rc);
13069 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13070
13071 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13072 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13073 return VINF_SUCCESS;
13074}
13075
13076
13077/**
13078 * VM-exit exception handler for \#DB (Debug exception).
13079 */
13080static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13081{
13082 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13083 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13084 Log6(("XcptDB\n"));
13085
13086 /*
13087 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13088 * for processing.
13089 */
13090 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13091 AssertRCReturn(rc, rc);
13092
13093 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13094 uint64_t uDR6 = X86_DR6_INIT_VAL;
13095 uDR6 |= ( pVmxTransient->uExitQualification
13096 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13097
13098 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13099 if (rc == VINF_EM_RAW_GUEST_TRAP)
13100 {
13101 /*
13102 * The exception was for the guest. Update DR6, DR7.GD and
13103 * IA32_DEBUGCTL.LBR before forwarding it.
13104 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13105 */
13106 VMMRZCallRing3Disable(pVCpu);
13107 HM_DISABLE_PREEMPT();
13108
13109 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13110 pMixedCtx->dr[6] |= uDR6;
13111 if (CPUMIsGuestDebugStateActive(pVCpu))
13112 ASMSetDR6(pMixedCtx->dr[6]);
13113
13114 HM_RESTORE_PREEMPT();
13115 VMMRZCallRing3Enable(pVCpu);
13116
13117 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13118 AssertRCReturn(rc, rc);
13119
13120 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13121 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13122
13123 /* Paranoia. */
13124 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13125 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13126
13127 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13128 AssertRCReturn(rc, rc);
13129
13130 /*
13131 * Raise #DB in the guest.
13132 *
13133 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13134 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13135 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13136 *
13137 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13138 */
13139 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13140 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13141 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13142 AssertRCReturn(rc, rc);
13143 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13144 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13145 return VINF_SUCCESS;
13146 }
13147
13148 /*
13149 * Not a guest trap, must be a hypervisor related debug event then.
13150 * Update DR6 in case someone is interested in it.
13151 */
13152 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13153 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13154 CPUMSetHyperDR6(pVCpu, uDR6);
13155
13156 return rc;
13157}
13158
13159
13160/**
13161 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13162 * point exception).
13163 */
13164static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13165{
13166 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13167
13168 /* We require CR0 and EFER. EFER is always up-to-date. */
13169 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13170 AssertRCReturn(rc, rc);
13171
13172 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13173 VMMRZCallRing3Disable(pVCpu);
13174 HM_DISABLE_PREEMPT();
13175
13176 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13177 if (pVmxTransient->fWasGuestFPUStateActive)
13178 {
13179 rc = VINF_EM_RAW_GUEST_TRAP;
13180 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13181 }
13182 else
13183 {
13184#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13185 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13186#endif
13187 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13188 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13189 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13190 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13191 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13192 }
13193
13194 HM_RESTORE_PREEMPT();
13195 VMMRZCallRing3Enable(pVCpu);
13196
13197 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13198 {
13199 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13200 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13202 pVCpu->hm.s.fPreloadGuestFpu = true;
13203 }
13204 else
13205 {
13206 /* Forward #NM to the guest. */
13207 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13208 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13209 AssertRCReturn(rc, rc);
13210 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13211 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13212 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13213 }
13214
13215 return VINF_SUCCESS;
13216}
13217
13218
13219/**
13220 * VM-exit exception handler for \#GP (General-protection exception).
13221 *
13222 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13223 */
13224static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13225{
13226 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13228
13229 int rc;
13230 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13231 { /* likely */ }
13232 else
13233 {
13234#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13235 Assert(pVCpu->hm.s.fUsingDebugLoop);
13236#endif
13237 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13238 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13239 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13240 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13241 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13242 AssertRCReturn(rc, rc);
13243 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13244 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13245 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13246 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13247 return rc;
13248 }
13249
13250 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13251 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13252
13253 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13254 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13255 AssertRCReturn(rc, rc);
13256
13257 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13258 uint32_t cbOp = 0;
13259 PVM pVM = pVCpu->CTX_SUFF(pVM);
13260 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13261 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13262 if (RT_SUCCESS(rc))
13263 {
13264 rc = VINF_SUCCESS;
13265 Assert(cbOp == pDis->cbInstr);
13266 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13267 switch (pDis->pCurInstr->uOpcode)
13268 {
13269 case OP_CLI:
13270 {
13271 pMixedCtx->eflags.Bits.u1IF = 0;
13272 pMixedCtx->eflags.Bits.u1RF = 0;
13273 pMixedCtx->rip += pDis->cbInstr;
13274 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13275 if ( !fDbgStepping
13276 && pMixedCtx->eflags.Bits.u1TF)
13277 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13278 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13279 break;
13280 }
13281
13282 case OP_STI:
13283 {
13284 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13285 pMixedCtx->eflags.Bits.u1IF = 1;
13286 pMixedCtx->eflags.Bits.u1RF = 0;
13287 pMixedCtx->rip += pDis->cbInstr;
13288 if (!fOldIF)
13289 {
13290 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13291 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13292 }
13293 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13294 if ( !fDbgStepping
13295 && pMixedCtx->eflags.Bits.u1TF)
13296 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13297 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13298 break;
13299 }
13300
13301 case OP_HLT:
13302 {
13303 rc = VINF_EM_HALT;
13304 pMixedCtx->rip += pDis->cbInstr;
13305 pMixedCtx->eflags.Bits.u1RF = 0;
13306 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13307 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13308 break;
13309 }
13310
13311 case OP_POPF:
13312 {
13313 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13314 uint32_t cbParm;
13315 uint32_t uMask;
13316 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13317 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13318 {
13319 cbParm = 4;
13320 uMask = 0xffffffff;
13321 }
13322 else
13323 {
13324 cbParm = 2;
13325 uMask = 0xffff;
13326 }
13327
13328 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13329 RTGCPTR GCPtrStack = 0;
13330 X86EFLAGS Eflags;
13331 Eflags.u32 = 0;
13332 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13333 &GCPtrStack);
13334 if (RT_SUCCESS(rc))
13335 {
13336 Assert(sizeof(Eflags.u32) >= cbParm);
13337 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13338 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13339 }
13340 if (RT_FAILURE(rc))
13341 {
13342 rc = VERR_EM_INTERPRETER;
13343 break;
13344 }
13345 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13346 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13347 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13348 pMixedCtx->esp += cbParm;
13349 pMixedCtx->esp &= uMask;
13350 pMixedCtx->rip += pDis->cbInstr;
13351 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13352 | HM_CHANGED_GUEST_RSP
13353 | HM_CHANGED_GUEST_RFLAGS);
13354 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13355 POPF restores EFLAGS.TF. */
13356 if ( !fDbgStepping
13357 && fGstStepping)
13358 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13360 break;
13361 }
13362
13363 case OP_PUSHF:
13364 {
13365 uint32_t cbParm;
13366 uint32_t uMask;
13367 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13368 {
13369 cbParm = 4;
13370 uMask = 0xffffffff;
13371 }
13372 else
13373 {
13374 cbParm = 2;
13375 uMask = 0xffff;
13376 }
13377
13378 /* Get the stack pointer & push the contents of eflags onto the stack. */
13379 RTGCPTR GCPtrStack = 0;
13380 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13381 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13382 if (RT_FAILURE(rc))
13383 {
13384 rc = VERR_EM_INTERPRETER;
13385 break;
13386 }
13387 X86EFLAGS Eflags = pMixedCtx->eflags;
13388 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13389 Eflags.Bits.u1RF = 0;
13390 Eflags.Bits.u1VM = 0;
13391
13392 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13393 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13394 {
13395 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13396 rc = VERR_EM_INTERPRETER;
13397 break;
13398 }
13399 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13400 pMixedCtx->esp -= cbParm;
13401 pMixedCtx->esp &= uMask;
13402 pMixedCtx->rip += pDis->cbInstr;
13403 pMixedCtx->eflags.Bits.u1RF = 0;
13404 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13405 | HM_CHANGED_GUEST_RSP
13406 | HM_CHANGED_GUEST_RFLAGS);
13407 if ( !fDbgStepping
13408 && pMixedCtx->eflags.Bits.u1TF)
13409 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13411 break;
13412 }
13413
13414 case OP_IRET:
13415 {
13416 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13417 * instruction reference. */
13418 RTGCPTR GCPtrStack = 0;
13419 uint32_t uMask = 0xffff;
13420 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13421 uint16_t aIretFrame[3];
13422 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13423 {
13424 rc = VERR_EM_INTERPRETER;
13425 break;
13426 }
13427 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13428 &GCPtrStack);
13429 if (RT_SUCCESS(rc))
13430 {
13431 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13432 PGMACCESSORIGIN_HM));
13433 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13434 }
13435 if (RT_FAILURE(rc))
13436 {
13437 rc = VERR_EM_INTERPRETER;
13438 break;
13439 }
13440 pMixedCtx->eip = 0;
13441 pMixedCtx->ip = aIretFrame[0];
13442 pMixedCtx->cs.Sel = aIretFrame[1];
13443 pMixedCtx->cs.ValidSel = aIretFrame[1];
13444 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13445 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13446 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13447 pMixedCtx->sp += sizeof(aIretFrame);
13448 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13449 | HM_CHANGED_GUEST_SEGMENT_REGS
13450 | HM_CHANGED_GUEST_RSP
13451 | HM_CHANGED_GUEST_RFLAGS);
13452 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13453 if ( !fDbgStepping
13454 && fGstStepping)
13455 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13456 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13458 break;
13459 }
13460
13461 case OP_INT:
13462 {
13463 uint16_t uVector = pDis->Param1.uValue & 0xff;
13464 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13465 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13467 break;
13468 }
13469
13470 case OP_INTO:
13471 {
13472 if (pMixedCtx->eflags.Bits.u1OF)
13473 {
13474 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13475 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13477 }
13478 else
13479 {
13480 pMixedCtx->eflags.Bits.u1RF = 0;
13481 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13482 }
13483 break;
13484 }
13485
13486 default:
13487 {
13488 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13489 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13490 EMCODETYPE_SUPERVISOR);
13491 rc = VBOXSTRICTRC_VAL(rc2);
13492 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13493 /** @todo We have to set pending-debug exceptions here when the guest is
13494 * single-stepping depending on the instruction that was interpreted. */
13495 Log4(("#GP rc=%Rrc\n", rc));
13496 break;
13497 }
13498 }
13499 }
13500 else
13501 rc = VERR_EM_INTERPRETER;
13502
13503 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13504 ("#GP Unexpected rc=%Rrc\n", rc));
13505 return rc;
13506}
13507
13508
13509/**
13510 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13511 * the exception reported in the VMX transient structure back into the VM.
13512 *
13513 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13514 * up-to-date.
13515 */
13516static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13517{
13518 RT_NOREF_PV(pMixedCtx);
13519 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13520#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13521 Assert(pVCpu->hm.s.fUsingDebugLoop);
13522#endif
13523
13524 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13525 hmR0VmxCheckExitDueToEventDelivery(). */
13526 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13527 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13528 AssertRCReturn(rc, rc);
13529 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13530
13531#ifdef DEBUG_ramshankar
13532 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13533 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13534 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13535#endif
13536
13537 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13538 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13539 return VINF_SUCCESS;
13540}
13541
13542
13543/**
13544 * VM-exit exception handler for \#PF (Page-fault exception).
13545 */
13546static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13547{
13548 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13549 PVM pVM = pVCpu->CTX_SUFF(pVM);
13550 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13551 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13552 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13553 AssertRCReturn(rc, rc);
13554
13555 if (!pVM->hm.s.fNestedPaging)
13556 { /* likely */ }
13557 else
13558 {
13559#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13560 Assert(pVCpu->hm.s.fUsingDebugLoop);
13561#endif
13562 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13563 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13564 {
13565 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13566 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13567 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13568 }
13569 else
13570 {
13571 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13572 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13573 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13574 }
13575 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13576 return rc;
13577 }
13578
13579 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13580 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13581 if (pVmxTransient->fVectoringPF)
13582 {
13583 Assert(pVCpu->hm.s.Event.fPending);
13584 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13585 }
13586
13587 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13588 AssertRCReturn(rc, rc);
13589
13590 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13591 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13592
13593 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13594 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13595 (RTGCPTR)pVmxTransient->uExitQualification);
13596
13597 Log4(("#PF: rc=%Rrc\n", rc));
13598 if (rc == VINF_SUCCESS)
13599 {
13600#if 0
13601 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13602 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13603 * memory? We don't update the whole state here... */
13604 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13605 | HM_CHANGED_GUEST_RSP
13606 | HM_CHANGED_GUEST_RFLAGS
13607 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13608#else
13609 /*
13610 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13611 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13612 */
13613 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13614 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13615#endif
13616 TRPMResetTrap(pVCpu);
13617 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13618 return rc;
13619 }
13620
13621 if (rc == VINF_EM_RAW_GUEST_TRAP)
13622 {
13623 if (!pVmxTransient->fVectoringDoublePF)
13624 {
13625 /* It's a guest page fault and needs to be reflected to the guest. */
13626 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13627 TRPMResetTrap(pVCpu);
13628 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13629 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13630 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13631 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13632 }
13633 else
13634 {
13635 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13636 TRPMResetTrap(pVCpu);
13637 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13638 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13639 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13640 }
13641
13642 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13643 return VINF_SUCCESS;
13644 }
13645
13646 TRPMResetTrap(pVCpu);
13647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13648 return rc;
13649}
13650
13651/** @} */
13652
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette