VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 61455

Last change on this file since 61455 was 61455, checked in by vboxsync, 9 years ago

HMR0EnsureCompleteBasicContext: Mark the guest state as changed.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 576.6 KB
Line 
1/* $Id: HMVMXR0.cpp 61455 2016-06-03 18:15:42Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST));
1476 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477 {
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 */
2912 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2913 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2914#endif
2915
2916 /*
2917 * Host DS, ES, FS and GS segment registers.
2918 */
2919#if HC_ARCH_BITS == 64
2920 RTSEL uSelDS = ASMGetDS();
2921 RTSEL uSelES = ASMGetES();
2922 RTSEL uSelFS = ASMGetFS();
2923 RTSEL uSelGS = ASMGetGS();
2924#else
2925 RTSEL uSelDS = 0;
2926 RTSEL uSelES = 0;
2927 RTSEL uSelFS = 0;
2928 RTSEL uSelGS = 0;
2929#endif
2930
2931 /* Recalculate which host-state bits need to be manually restored. */
2932 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2933
2934 /*
2935 * Host CS and SS segment registers.
2936 */
2937 RTSEL uSelCS = ASMGetCS();
2938 RTSEL uSelSS = ASMGetSS();
2939
2940 /*
2941 * Host TR segment register.
2942 */
2943 RTSEL uSelTR = ASMGetTR();
2944
2945#if HC_ARCH_BITS == 64
2946 /*
2947 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2948 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2949 */
2950 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2951 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2952 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2953 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2954# undef VMXLOCAL_ADJUST_HOST_SEG
2955#endif
2956
2957 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2958 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2959 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2960 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2961 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2962 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2963 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2964 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2965 Assert(uSelCS);
2966 Assert(uSelTR);
2967
2968 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2969#if 0
2970 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2971 Assert(uSelSS != 0);
2972#endif
2973
2974 /* Write these host selector fields into the host-state area in the VMCS. */
2975 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2976 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2977#if HC_ARCH_BITS == 64
2978 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2979 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2980 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2981 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2982#else
2983 NOREF(uSelDS);
2984 NOREF(uSelES);
2985 NOREF(uSelFS);
2986 NOREF(uSelGS);
2987#endif
2988 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2989 AssertRCReturn(rc, rc);
2990
2991 /*
2992 * Host GDTR and IDTR.
2993 */
2994 RTGDTR Gdtr;
2995 RTIDTR Idtr;
2996 RT_ZERO(Gdtr);
2997 RT_ZERO(Idtr);
2998 ASMGetGDTR(&Gdtr);
2999 ASMGetIDTR(&Idtr);
3000 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3001 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3002 AssertRCReturn(rc, rc);
3003
3004#if HC_ARCH_BITS == 64
3005 /*
3006 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3007 * maximum limit (0xffff) on every VM-exit.
3008 */
3009 if (Gdtr.cbGdt != 0xffff)
3010 {
3011 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3012 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3013 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3014 }
3015
3016 /*
3017 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3018 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3019 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3020 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3021 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3022 * hosts where we are pretty sure it won't cause trouble.
3023 */
3024# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3025 if (Idtr.cbIdt < 0x0fff)
3026# else
3027 if (Idtr.cbIdt != 0xffff)
3028# endif
3029 {
3030 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3031 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3032 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3033 }
3034#endif
3035
3036 /*
3037 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3038 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3039 */
3040 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3041 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3042 VERR_VMX_INVALID_HOST_STATE);
3043
3044 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3045#if HC_ARCH_BITS == 64
3046 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3047
3048 /*
3049 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3050 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3051 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3052 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3053 *
3054 * [1] See Intel spec. 3.5 "System Descriptor Types".
3055 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3056 */
3057 Assert(pDesc->System.u4Type == 11);
3058 if ( pDesc->System.u16LimitLow != 0x67
3059 || pDesc->System.u4LimitHigh)
3060 {
3061 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3062 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3063 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3064 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3065 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3066
3067 /* Store the GDTR here as we need it while restoring TR. */
3068 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3069 }
3070#else
3071 NOREF(pVM);
3072 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073#endif
3074 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /*
3078 * Host FS base and GS base.
3079 */
3080#if HC_ARCH_BITS == 64
3081 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085 AssertRCReturn(rc, rc);
3086
3087 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092#endif
3093 return rc;
3094}
3095
3096
3097/**
3098 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3099 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3100 * the host after every successful VM-exit.
3101 *
3102 * @returns VBox status code.
3103 * @param pVM The cross context VM structure.
3104 * @param pVCpu The cross context virtual CPU structure.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3109{
3110 NOREF(pVM);
3111
3112 AssertPtr(pVCpu);
3113 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3114
3115 /*
3116 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3117 * rather than swapping them on every VM-entry.
3118 */
3119 hmR0VmxLazySaveHostMsrs(pVCpu);
3120
3121 /*
3122 * Host Sysenter MSRs.
3123 */
3124 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3125#if HC_ARCH_BITS == 32
3126 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3127 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3128#else
3129 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3130 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3131#endif
3132 AssertRCReturn(rc, rc);
3133
3134 /*
3135 * Host EFER MSR.
3136 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3137 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3138 */
3139 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3140 {
3141 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3142 AssertRCReturn(rc, rc);
3143 }
3144
3145 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3146 * hmR0VmxLoadGuestExitCtls() !! */
3147
3148 return rc;
3149}
3150
3151
3152/**
3153 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3154 *
3155 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3156 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3157 * hmR0VMxLoadGuestEntryCtls().
3158 *
3159 * @returns true if we need to load guest EFER, false otherwise.
3160 * @param pVCpu The cross context virtual CPU structure.
3161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3162 * out-of-sync. Make sure to update the required fields
3163 * before using them.
3164 *
3165 * @remarks Requires EFER, CR4.
3166 * @remarks No-long-jump zone!!!
3167 */
3168static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3169{
3170#ifdef HMVMX_ALWAYS_SWAP_EFER
3171 return true;
3172#endif
3173
3174#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3175 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3176 if (CPUMIsGuestInLongMode(pVCpu))
3177 return false;
3178#endif
3179
3180 PVM pVM = pVCpu->CTX_SUFF(pVM);
3181 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3182 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3183
3184 /*
3185 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3186 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3187 */
3188 if ( CPUMIsGuestInLongMode(pVCpu)
3189 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3190 {
3191 return true;
3192 }
3193
3194 /*
3195 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3196 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3197 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3198 */
3199 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3200 && (pMixedCtx->cr0 & X86_CR0_PG)
3201 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3202 {
3203 /* Assert that host is PAE capable. */
3204 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3205 return true;
3206 }
3207
3208 /** @todo Check the latest Intel spec. for any other bits,
3209 * like SMEP/SMAP? */
3210 return false;
3211}
3212
3213
3214/**
3215 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3216 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3217 * controls".
3218 *
3219 * @returns VBox status code.
3220 * @param pVCpu The cross context virtual CPU structure.
3221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3222 * out-of-sync. Make sure to update the required fields
3223 * before using them.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3229{
3230 int rc = VINF_SUCCESS;
3231 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3232 {
3233 PVM pVM = pVCpu->CTX_SUFF(pVM);
3234 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3235 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3236
3237 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3238 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3239
3240 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3241 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3242 {
3243 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3244 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3245 }
3246 else
3247 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3248
3249 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3250 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3251 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3252 {
3253 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3254 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3255 }
3256
3257 /*
3258 * The following should -not- be set (since we're not in SMM mode):
3259 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3260 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3261 */
3262
3263 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3264 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3265
3266 if ((val & zap) != val)
3267 {
3268 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3269 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3270 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3272 }
3273
3274 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3275 AssertRCReturn(rc, rc);
3276
3277 pVCpu->hm.s.vmx.u32EntryCtls = val;
3278 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3279 }
3280 return rc;
3281}
3282
3283
3284/**
3285 * Sets up the VM-exit controls in the VMCS.
3286 *
3287 * @returns VBox status code.
3288 * @param pVCpu The cross context virtual CPU structure.
3289 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3290 * out-of-sync. Make sure to update the required fields
3291 * before using them.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3296{
3297 NOREF(pMixedCtx);
3298
3299 int rc = VINF_SUCCESS;
3300 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3301 {
3302 PVM pVM = pVCpu->CTX_SUFF(pVM);
3303 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3304 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3305
3306 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3307 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3308
3309 /*
3310 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3311 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3312 */
3313#if HC_ARCH_BITS == 64
3314 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3315 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3316#else
3317 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3318 {
3319 /* The switcher returns to long mode, EFER is managed by the switcher. */
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322 }
3323 else
3324 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3325#endif
3326
3327 /* If the newer VMCS fields for managing EFER exists, use it. */
3328 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3329 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3330 {
3331 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3332 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3333 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3334 }
3335
3336 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3337 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3338
3339 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3340 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3341 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3342
3343 if ( pVM->hm.s.vmx.fUsePreemptTimer
3344 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3345 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3346
3347 if ((val & zap) != val)
3348 {
3349 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3350 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3351 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3352 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3353 }
3354
3355 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3356 AssertRCReturn(rc, rc);
3357
3358 pVCpu->hm.s.vmx.u32ExitCtls = val;
3359 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3360 }
3361 return rc;
3362}
3363
3364
3365/**
3366 * Sets the TPR threshold in the VMCS.
3367 *
3368 * @returns VBox status code.
3369 * @param pVCpu The cross context virtual CPU structure.
3370 * @param u32TprThreshold The TPR threshold (task-priority class only).
3371 */
3372DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3373{
3374 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3375 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3376 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3377}
3378
3379
3380/**
3381 * Loads the guest APIC and related state.
3382 *
3383 * @returns VBox status code.
3384 * @param pVCpu The cross context virtual CPU structure.
3385 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3386 * out-of-sync. Make sure to update the required fields
3387 * before using them.
3388 */
3389DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3390{
3391 NOREF(pMixedCtx);
3392
3393 int rc = VINF_SUCCESS;
3394 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3395 {
3396 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3397 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3398 {
3399 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3400
3401 bool fPendingIntr = false;
3402 uint8_t u8Tpr = 0;
3403 uint8_t u8PendingIntr = 0;
3404 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3405 AssertRCReturn(rc, rc);
3406
3407 /*
3408 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3409 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3410 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3411 * the interrupt when we VM-exit for other reasons.
3412 */
3413 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3414 uint32_t u32TprThreshold = 0;
3415 if (fPendingIntr)
3416 {
3417 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3418 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3419 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3420 if (u8PendingPriority <= u8TprPriority)
3421 u32TprThreshold = u8PendingPriority;
3422 else
3423 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3424 }
3425
3426 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3427 AssertRCReturn(rc, rc);
3428 }
3429
3430 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3431 }
3432 return rc;
3433}
3434
3435
3436/**
3437 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3438 *
3439 * @returns Guest's interruptibility-state.
3440 * @param pVCpu The cross context virtual CPU structure.
3441 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3442 * out-of-sync. Make sure to update the required fields
3443 * before using them.
3444 *
3445 * @remarks No-long-jump zone!!!
3446 */
3447DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3448{
3449 /*
3450 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3451 */
3452 uint32_t uIntrState = 0;
3453 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3454 {
3455 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3456 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3457 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3458 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3459 {
3460 if (pMixedCtx->eflags.Bits.u1IF)
3461 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3462 else
3463 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3464 }
3465 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466 {
3467 /*
3468 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3469 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3470 */
3471 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3472 }
3473 }
3474
3475 /*
3476 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3477 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3478 * setting this would block host-NMIs and IRET will not clear the blocking.
3479 *
3480 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3481 */
3482 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3483 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3484 {
3485 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3486 }
3487
3488 return uIntrState;
3489}
3490
3491
3492/**
3493 * Loads the guest's interruptibility-state into the guest-state area in the
3494 * VMCS.
3495 *
3496 * @returns VBox status code.
3497 * @param pVCpu The cross context virtual CPU structure.
3498 * @param uIntrState The interruptibility-state to set.
3499 */
3500static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3501{
3502 NOREF(pVCpu);
3503 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3504 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3505 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3506 AssertRC(rc);
3507 return rc;
3508}
3509
3510
3511/**
3512 * Loads the exception intercepts required for guest execution in the VMCS.
3513 *
3514 * @returns VBox status code.
3515 * @param pVCpu The cross context virtual CPU structure.
3516 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3517 * out-of-sync. Make sure to update the required fields
3518 * before using them.
3519 */
3520static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3521{
3522 NOREF(pMixedCtx);
3523 int rc = VINF_SUCCESS;
3524 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3525 {
3526 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3527 if (pVCpu->hm.s.fGIMTrapXcptUD)
3528 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3529#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530 else
3531 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3532#endif
3533
3534 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3535 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3536
3537 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3538 AssertRCReturn(rc, rc);
3539
3540 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3541 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3542 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3543 }
3544 return rc;
3545}
3546
3547
3548/**
3549 * Loads the guest's RIP into the guest-state area in the VMCS.
3550 *
3551 * @returns VBox status code.
3552 * @param pVCpu The cross context virtual CPU structure.
3553 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3554 * out-of-sync. Make sure to update the required fields
3555 * before using them.
3556 *
3557 * @remarks No-long-jump zone!!!
3558 */
3559static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3560{
3561 int rc = VINF_SUCCESS;
3562 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3563 {
3564 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3565 AssertRCReturn(rc, rc);
3566
3567 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3568 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3569 HMCPU_CF_VALUE(pVCpu)));
3570 }
3571 return rc;
3572}
3573
3574
3575/**
3576 * Loads the guest's RSP into the guest-state area in the VMCS.
3577 *
3578 * @returns VBox status code.
3579 * @param pVCpu The cross context virtual CPU structure.
3580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3581 * out-of-sync. Make sure to update the required fields
3582 * before using them.
3583 *
3584 * @remarks No-long-jump zone!!!
3585 */
3586static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3587{
3588 int rc = VINF_SUCCESS;
3589 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3590 {
3591 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3592 AssertRCReturn(rc, rc);
3593
3594 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3595 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3596 }
3597 return rc;
3598}
3599
3600
3601/**
3602 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3603 *
3604 * @returns VBox status code.
3605 * @param pVCpu The cross context virtual CPU structure.
3606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3607 * out-of-sync. Make sure to update the required fields
3608 * before using them.
3609 *
3610 * @remarks No-long-jump zone!!!
3611 */
3612static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3613{
3614 int rc = VINF_SUCCESS;
3615 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3616 {
3617 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3618 Let us assert it as such and use 32-bit VMWRITE. */
3619 Assert(!(pMixedCtx->rflags.u64 >> 32));
3620 X86EFLAGS Eflags = pMixedCtx->eflags;
3621 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3622 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3623 * These will never be cleared/set, unless some other part of the VMM
3624 * code is buggy - in which case we're better of finding and fixing
3625 * those bugs than hiding them. */
3626 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3627 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3628 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3629 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3630
3631 /*
3632 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3633 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3634 */
3635 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3636 {
3637 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3638 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3639 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3640 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3641 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3642 }
3643
3644 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3645 AssertRCReturn(rc, rc);
3646
3647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3648 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3649 }
3650 return rc;
3651}
3652
3653
3654/**
3655 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu The cross context virtual CPU structure.
3659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3660 * out-of-sync. Make sure to update the required fields
3661 * before using them.
3662 *
3663 * @remarks No-long-jump zone!!!
3664 */
3665DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3666{
3667 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3668 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3669 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3670 AssertRCReturn(rc, rc);
3671 return rc;
3672}
3673
3674
3675/**
3676 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3677 * CR0 is partially shared with the host and we have to consider the FPU bits.
3678 *
3679 * @returns VBox status code.
3680 * @param pVCpu The cross context virtual CPU structure.
3681 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3682 * out-of-sync. Make sure to update the required fields
3683 * before using them.
3684 *
3685 * @remarks No-long-jump zone!!!
3686 */
3687static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3688{
3689 /*
3690 * Guest CR0.
3691 * Guest FPU.
3692 */
3693 int rc = VINF_SUCCESS;
3694 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3695 {
3696 Assert(!(pMixedCtx->cr0 >> 32));
3697 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3698 PVM pVM = pVCpu->CTX_SUFF(pVM);
3699
3700 /* The guest's view (read access) of its CR0 is unblemished. */
3701 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3702 AssertRCReturn(rc, rc);
3703 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3704
3705 /* Setup VT-x's view of the guest CR0. */
3706 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3707 if (pVM->hm.s.fNestedPaging)
3708 {
3709 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3710 {
3711 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3712 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3713 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3714 }
3715 else
3716 {
3717 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3718 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3719 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3720 }
3721
3722 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3723 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3724 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3725
3726 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3727 AssertRCReturn(rc, rc);
3728 }
3729 else
3730 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3731
3732 /*
3733 * Guest FPU bits.
3734 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3735 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3736 */
3737 u32GuestCR0 |= X86_CR0_NE;
3738 bool fInterceptNM = false;
3739 if (CPUMIsGuestFPUStateActive(pVCpu))
3740 {
3741 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3742 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3743 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3744 }
3745 else
3746 {
3747 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3748 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3749 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3750 }
3751
3752 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3753 bool fInterceptMF = false;
3754 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3755 fInterceptMF = true;
3756
3757 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3758 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3759 {
3760 Assert(PDMVmmDevHeapIsEnabled(pVM));
3761 Assert(pVM->hm.s.vmx.pRealModeTSS);
3762 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3763 fInterceptNM = true;
3764 fInterceptMF = true;
3765 }
3766 else
3767 {
3768 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3769 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3770 }
3771 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3772
3773 if (fInterceptNM)
3774 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3777
3778 if (fInterceptMF)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3782
3783 /* Additional intercepts for debugging, define these yourself explicitly. */
3784#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3785 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3786 | RT_BIT(X86_XCPT_BP)
3787 | RT_BIT(X86_XCPT_DE)
3788 | RT_BIT(X86_XCPT_NM)
3789 | RT_BIT(X86_XCPT_TS)
3790 | RT_BIT(X86_XCPT_UD)
3791 | RT_BIT(X86_XCPT_NP)
3792 | RT_BIT(X86_XCPT_SS)
3793 | RT_BIT(X86_XCPT_GP)
3794 | RT_BIT(X86_XCPT_PF)
3795 | RT_BIT(X86_XCPT_MF)
3796 ;
3797#elif defined(HMVMX_ALWAYS_TRAP_PF)
3798 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3799#endif
3800
3801 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3802
3803 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3804 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3805 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3806 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3807 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3808 else
3809 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3810
3811 u32GuestCR0 |= uSetCR0;
3812 u32GuestCR0 &= uZapCR0;
3813 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3814
3815 /* Write VT-x's view of the guest CR0 into the VMCS. */
3816 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3817 AssertRCReturn(rc, rc);
3818 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3819 uZapCR0));
3820
3821 /*
3822 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3823 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3824 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3825 */
3826 uint32_t u32CR0Mask = 0;
3827 u32CR0Mask = X86_CR0_PE
3828 | X86_CR0_NE
3829 | X86_CR0_WP
3830 | X86_CR0_PG
3831 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3832 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3833 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3834
3835 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3836 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3837 * and @bugref{6944}. */
3838#if 0
3839 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3840 u32CR0Mask &= ~X86_CR0_PE;
3841#endif
3842 if (pVM->hm.s.fNestedPaging)
3843 u32CR0Mask &= ~X86_CR0_WP;
3844
3845 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3846 if (fInterceptNM)
3847 {
3848 u32CR0Mask |= X86_CR0_TS
3849 | X86_CR0_MP;
3850 }
3851
3852 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3853 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3854 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3855 AssertRCReturn(rc, rc);
3856 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3857
3858 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3859 }
3860 return rc;
3861}
3862
3863
3864/**
3865 * Loads the guest control registers (CR3, CR4) into the guest-state area
3866 * in the VMCS.
3867 *
3868 * @returns VBox strict status code.
3869 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3870 * without unrestricted guest access and the VMMDev is not presently
3871 * mapped (e.g. EFI32).
3872 *
3873 * @param pVCpu The cross context virtual CPU structure.
3874 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3875 * out-of-sync. Make sure to update the required fields
3876 * before using them.
3877 *
3878 * @remarks No-long-jump zone!!!
3879 */
3880static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3881{
3882 int rc = VINF_SUCCESS;
3883 PVM pVM = pVCpu->CTX_SUFF(pVM);
3884
3885 /*
3886 * Guest CR2.
3887 * It's always loaded in the assembler code. Nothing to do here.
3888 */
3889
3890 /*
3891 * Guest CR3.
3892 */
3893 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3894 {
3895 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3896 if (pVM->hm.s.fNestedPaging)
3897 {
3898 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3899
3900 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3901 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3902 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3903 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3904
3905 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3906 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3907 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3908
3909 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3910 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3911 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3912 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3913 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3914 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3915 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916
3917 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3918 AssertRCReturn(rc, rc);
3919 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3920
3921 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3922 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3923 {
3924 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3925 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3926 {
3927 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3928 AssertRCReturn(rc, rc);
3929 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3930 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3931 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3932 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3933 AssertRCReturn(rc, rc);
3934 }
3935
3936 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937 have Unrestricted Execution to handle the guest when it's not using paging. */
3938 GCPhysGuestCR3 = pMixedCtx->cr3;
3939 }
3940 else
3941 {
3942 /*
3943 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945 * EPT takes care of translating it to host-physical addresses.
3946 */
3947 RTGCPHYS GCPhys;
3948 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949
3950 /* We obtain it here every time as the guest could have relocated this PCI region. */
3951 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3952 if (RT_SUCCESS(rc))
3953 { /* likely */ }
3954 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3955 {
3956 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3957 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3958 }
3959 else
3960 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3961
3962 GCPhysGuestCR3 = GCPhys;
3963 }
3964
3965 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3966 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3967 }
3968 else
3969 {
3970 /* Non-nested paging case, just use the hypervisor's CR3. */
3971 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3972
3973 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3974 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3975 }
3976 AssertRCReturn(rc, rc);
3977
3978 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3979 }
3980
3981 /*
3982 * Guest CR4.
3983 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3984 */
3985 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3986 {
3987 Assert(!(pMixedCtx->cr4 >> 32));
3988 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3989
3990 /* The guest's view of its CR4 is unblemished. */
3991 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3992 AssertRCReturn(rc, rc);
3993 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3994
3995 /* Setup VT-x's view of the guest CR4. */
3996 /*
3997 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3998 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3999 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4000 */
4001 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4002 {
4003 Assert(pVM->hm.s.vmx.pRealModeTSS);
4004 Assert(PDMVmmDevHeapIsEnabled(pVM));
4005 u32GuestCR4 &= ~X86_CR4_VME;
4006 }
4007
4008 if (pVM->hm.s.fNestedPaging)
4009 {
4010 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4011 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4012 {
4013 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4014 u32GuestCR4 |= X86_CR4_PSE;
4015 /* Our identity mapping is a 32-bit page directory. */
4016 u32GuestCR4 &= ~X86_CR4_PAE;
4017 }
4018 /* else use guest CR4.*/
4019 }
4020 else
4021 {
4022 /*
4023 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4024 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4025 */
4026 switch (pVCpu->hm.s.enmShadowMode)
4027 {
4028 case PGMMODE_REAL: /* Real-mode. */
4029 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4030 case PGMMODE_32_BIT: /* 32-bit paging. */
4031 {
4032 u32GuestCR4 &= ~X86_CR4_PAE;
4033 break;
4034 }
4035
4036 case PGMMODE_PAE: /* PAE paging. */
4037 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4038 {
4039 u32GuestCR4 |= X86_CR4_PAE;
4040 break;
4041 }
4042
4043 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4044 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4045#ifdef VBOX_ENABLE_64_BITS_GUESTS
4046 break;
4047#endif
4048 default:
4049 AssertFailed();
4050 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4051 }
4052 }
4053
4054 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4055 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4056 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4057 u32GuestCR4 |= uSetCR4;
4058 u32GuestCR4 &= uZapCR4;
4059
4060 /* Write VT-x's view of the guest CR4 into the VMCS. */
4061 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4062 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4063 AssertRCReturn(rc, rc);
4064
4065 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4066 uint32_t u32CR4Mask = X86_CR4_VME
4067 | X86_CR4_PAE
4068 | X86_CR4_PGE
4069 | X86_CR4_PSE
4070 | X86_CR4_VMXE;
4071 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4072 u32CR4Mask |= X86_CR4_OSXSAVE;
4073 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4074 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4075 AssertRCReturn(rc, rc);
4076
4077 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4078 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4079
4080 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4081 }
4082 return rc;
4083}
4084
4085
4086/**
4087 * Loads the guest debug registers into the guest-state area in the VMCS.
4088 *
4089 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4090 *
4091 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4092 *
4093 * @returns VBox status code.
4094 * @param pVCpu The cross context virtual CPU structure.
4095 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4096 * out-of-sync. Make sure to update the required fields
4097 * before using them.
4098 *
4099 * @remarks No-long-jump zone!!!
4100 */
4101static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4102{
4103 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4104 return VINF_SUCCESS;
4105
4106#ifdef VBOX_STRICT
4107 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4108 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4109 {
4110 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4111 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4112 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4113 }
4114#endif
4115
4116 int rc;
4117 PVM pVM = pVCpu->CTX_SUFF(pVM);
4118 bool fSteppingDB = false;
4119 bool fInterceptMovDRx = false;
4120 if (pVCpu->hm.s.fSingleInstruction)
4121 {
4122 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4123 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4124 {
4125 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4126 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4127 AssertRCReturn(rc, rc);
4128 Assert(fSteppingDB == false);
4129 }
4130 else
4131 {
4132 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4133 pVCpu->hm.s.fClearTrapFlag = true;
4134 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4135 fSteppingDB = true;
4136 }
4137 }
4138
4139 if ( fSteppingDB
4140 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4141 {
4142 /*
4143 * Use the combined guest and host DRx values found in the hypervisor
4144 * register set because the debugger has breakpoints active or someone
4145 * is single stepping on the host side without a monitor trap flag.
4146 *
4147 * Note! DBGF expects a clean DR6 state before executing guest code.
4148 */
4149#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4150 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4151 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4152 {
4153 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4154 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4155 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4156 }
4157 else
4158#endif
4159 if (!CPUMIsHyperDebugStateActive(pVCpu))
4160 {
4161 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4162 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4163 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4164 }
4165
4166 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4167 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4168 AssertRCReturn(rc, rc);
4169
4170 pVCpu->hm.s.fUsingHyperDR7 = true;
4171 fInterceptMovDRx = true;
4172 }
4173 else
4174 {
4175 /*
4176 * If the guest has enabled debug registers, we need to load them prior to
4177 * executing guest code so they'll trigger at the right time.
4178 */
4179 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4180 {
4181#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4182 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4183 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4184 {
4185 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4186 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4187 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4188 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4189 }
4190 else
4191#endif
4192 if (!CPUMIsGuestDebugStateActive(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 Assert(!fInterceptMovDRx);
4200 }
4201 /*
4202 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4203 * must intercept #DB in order to maintain a correct DR6 guest value, and
4204 * because we need to intercept it to prevent nested #DBs from hanging the
4205 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4206 */
4207#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4208 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4209 && !CPUMIsGuestDebugStateActive(pVCpu))
4210#else
4211 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4212#endif
4213 {
4214 fInterceptMovDRx = true;
4215 }
4216
4217 /* Update guest DR7. */
4218 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4219 AssertRCReturn(rc, rc);
4220
4221 pVCpu->hm.s.fUsingHyperDR7 = false;
4222 }
4223
4224 /*
4225 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4226 */
4227 if (fInterceptMovDRx)
4228 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4229 else
4230 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4231 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4232 AssertRCReturn(rc, rc);
4233
4234 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4235 return VINF_SUCCESS;
4236}
4237
4238
4239#ifdef VBOX_STRICT
4240/**
4241 * Strict function to validate segment registers.
4242 *
4243 * @remarks ASSUMES CR0 is up to date.
4244 */
4245static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4246{
4247 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4248 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4249 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4250 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4251 && ( !CPUMIsGuestInRealModeEx(pCtx)
4252 && !CPUMIsGuestInV86ModeEx(pCtx)))
4253 {
4254 /* Protected mode checks */
4255 /* CS */
4256 Assert(pCtx->cs.Attr.n.u1Present);
4257 Assert(!(pCtx->cs.Attr.u & 0xf00));
4258 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4259 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4260 || !(pCtx->cs.Attr.n.u1Granularity));
4261 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4262 || (pCtx->cs.Attr.n.u1Granularity));
4263 /* CS cannot be loaded with NULL in protected mode. */
4264 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4265 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4266 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4267 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4268 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4269 else
4270 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4271 /* SS */
4272 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4273 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4274 if ( !(pCtx->cr0 & X86_CR0_PE)
4275 || pCtx->cs.Attr.n.u4Type == 3)
4276 {
4277 Assert(!pCtx->ss.Attr.n.u2Dpl);
4278 }
4279 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4280 {
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4283 Assert(pCtx->ss.Attr.n.u1Present);
4284 Assert(!(pCtx->ss.Attr.u & 0xf00));
4285 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4286 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4287 || !(pCtx->ss.Attr.n.u1Granularity));
4288 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4289 || (pCtx->ss.Attr.n.u1Granularity));
4290 }
4291 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4292 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4293 {
4294 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4295 Assert(pCtx->ds.Attr.n.u1Present);
4296 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4297 Assert(!(pCtx->ds.Attr.u & 0xf00));
4298 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4299 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4300 || !(pCtx->ds.Attr.n.u1Granularity));
4301 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4302 || (pCtx->ds.Attr.n.u1Granularity));
4303 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4304 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4305 }
4306 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4307 {
4308 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4309 Assert(pCtx->es.Attr.n.u1Present);
4310 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4311 Assert(!(pCtx->es.Attr.u & 0xf00));
4312 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4313 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4314 || !(pCtx->es.Attr.n.u1Granularity));
4315 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4316 || (pCtx->es.Attr.n.u1Granularity));
4317 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4318 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4319 }
4320 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4321 {
4322 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4323 Assert(pCtx->fs.Attr.n.u1Present);
4324 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4325 Assert(!(pCtx->fs.Attr.u & 0xf00));
4326 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4327 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4328 || !(pCtx->fs.Attr.n.u1Granularity));
4329 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4330 || (pCtx->fs.Attr.n.u1Granularity));
4331 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4332 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4333 }
4334 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4335 {
4336 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4337 Assert(pCtx->gs.Attr.n.u1Present);
4338 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4339 Assert(!(pCtx->gs.Attr.u & 0xf00));
4340 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4341 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4342 || !(pCtx->gs.Attr.n.u1Granularity));
4343 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4344 || (pCtx->gs.Attr.n.u1Granularity));
4345 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4346 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4347 }
4348 /* 64-bit capable CPUs. */
4349# if HC_ARCH_BITS == 64
4350 Assert(!(pCtx->cs.u64Base >> 32));
4351 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4352 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4353 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4354# endif
4355 }
4356 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4357 || ( CPUMIsGuestInRealModeEx(pCtx)
4358 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4359 {
4360 /* Real and v86 mode checks. */
4361 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4362 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4363 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4364 {
4365 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4366 }
4367 else
4368 {
4369 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4370 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4371 }
4372
4373 /* CS */
4374 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4375 Assert(pCtx->cs.u32Limit == 0xffff);
4376 Assert(u32CSAttr == 0xf3);
4377 /* SS */
4378 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4379 Assert(pCtx->ss.u32Limit == 0xffff);
4380 Assert(u32SSAttr == 0xf3);
4381 /* DS */
4382 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4383 Assert(pCtx->ds.u32Limit == 0xffff);
4384 Assert(u32DSAttr == 0xf3);
4385 /* ES */
4386 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4387 Assert(pCtx->es.u32Limit == 0xffff);
4388 Assert(u32ESAttr == 0xf3);
4389 /* FS */
4390 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4391 Assert(pCtx->fs.u32Limit == 0xffff);
4392 Assert(u32FSAttr == 0xf3);
4393 /* GS */
4394 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4395 Assert(pCtx->gs.u32Limit == 0xffff);
4396 Assert(u32GSAttr == 0xf3);
4397 /* 64-bit capable CPUs. */
4398# if HC_ARCH_BITS == 64
4399 Assert(!(pCtx->cs.u64Base >> 32));
4400 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4401 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4402 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4403# endif
4404 }
4405}
4406#endif /* VBOX_STRICT */
4407
4408
4409/**
4410 * Writes a guest segment register into the guest-state area in the VMCS.
4411 *
4412 * @returns VBox status code.
4413 * @param pVCpu The cross context virtual CPU structure.
4414 * @param idxSel Index of the selector in the VMCS.
4415 * @param idxLimit Index of the segment limit in the VMCS.
4416 * @param idxBase Index of the segment base in the VMCS.
4417 * @param idxAccess Index of the access rights of the segment in the VMCS.
4418 * @param pSelReg Pointer to the segment selector.
4419 *
4420 * @remarks No-long-jump zone!!!
4421 */
4422static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4423 uint32_t idxAccess, PCPUMSELREG pSelReg)
4424{
4425 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4426 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4427 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4428 AssertRCReturn(rc, rc);
4429
4430 uint32_t u32Access = pSelReg->Attr.u;
4431 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4432 {
4433 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4434 u32Access = 0xf3;
4435 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4436 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4437 }
4438 else
4439 {
4440 /*
4441 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4442 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4443 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4444 * loaded in protected-mode have their attribute as 0.
4445 */
4446 if (!u32Access)
4447 u32Access = X86DESCATTR_UNUSABLE;
4448 }
4449
4450 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4451 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4452 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4453
4454 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4455 AssertRCReturn(rc, rc);
4456 return rc;
4457}
4458
4459
4460/**
4461 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4462 * into the guest-state area in the VMCS.
4463 *
4464 * @returns VBox status code.
4465 * @param pVCpu The cross context virtual CPU structure.
4466 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4467 * out-of-sync. Make sure to update the required fields
4468 * before using them.
4469 *
4470 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4471 * @remarks No-long-jump zone!!!
4472 */
4473static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4474{
4475 int rc = VERR_INTERNAL_ERROR_5;
4476 PVM pVM = pVCpu->CTX_SUFF(pVM);
4477
4478 /*
4479 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4480 */
4481 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4482 {
4483 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4484 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4485 {
4486 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4487 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4488 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4489 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4490 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4491 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4492 }
4493
4494#ifdef VBOX_WITH_REM
4495 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4496 {
4497 Assert(pVM->hm.s.vmx.pRealModeTSS);
4498 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4499 if ( pVCpu->hm.s.vmx.fWasInRealMode
4500 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4501 {
4502 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4503 in real-mode (e.g. OpenBSD 4.0) */
4504 REMFlushTBs(pVM);
4505 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4506 pVCpu->hm.s.vmx.fWasInRealMode = false;
4507 }
4508 }
4509#endif
4510 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4511 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4512 AssertRCReturn(rc, rc);
4513 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4514 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4515 AssertRCReturn(rc, rc);
4516 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4517 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4518 AssertRCReturn(rc, rc);
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4520 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4523 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4526 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4527 AssertRCReturn(rc, rc);
4528
4529#ifdef VBOX_STRICT
4530 /* Validate. */
4531 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4532#endif
4533
4534 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4535 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4536 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4537 }
4538
4539 /*
4540 * Guest TR.
4541 */
4542 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4543 {
4544 /*
4545 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4546 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4547 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4548 */
4549 uint16_t u16Sel = 0;
4550 uint32_t u32Limit = 0;
4551 uint64_t u64Base = 0;
4552 uint32_t u32AccessRights = 0;
4553
4554 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4555 {
4556 u16Sel = pMixedCtx->tr.Sel;
4557 u32Limit = pMixedCtx->tr.u32Limit;
4558 u64Base = pMixedCtx->tr.u64Base;
4559 u32AccessRights = pMixedCtx->tr.Attr.u;
4560 }
4561 else
4562 {
4563 Assert(pVM->hm.s.vmx.pRealModeTSS);
4564 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4565
4566 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4567 RTGCPHYS GCPhys;
4568 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4569 AssertRCReturn(rc, rc);
4570
4571 X86DESCATTR DescAttr;
4572 DescAttr.u = 0;
4573 DescAttr.n.u1Present = 1;
4574 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4575
4576 u16Sel = 0;
4577 u32Limit = HM_VTX_TSS_SIZE;
4578 u64Base = GCPhys; /* in real-mode phys = virt. */
4579 u32AccessRights = DescAttr.u;
4580 }
4581
4582 /* Validate. */
4583 Assert(!(u16Sel & RT_BIT(2)));
4584 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4585 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4586 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4587 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4588 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4589 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4590 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4591 Assert( (u32Limit & 0xfff) == 0xfff
4592 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4593 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4594 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4595
4596 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4597 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4598 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4599 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4600 AssertRCReturn(rc, rc);
4601
4602 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4603 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4604 }
4605
4606 /*
4607 * Guest GDTR.
4608 */
4609 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4610 {
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4612 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4613 AssertRCReturn(rc, rc);
4614
4615 /* Validate. */
4616 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4617
4618 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4619 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4620 }
4621
4622 /*
4623 * Guest LDTR.
4624 */
4625 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4626 {
4627 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4628 uint32_t u32Access = 0;
4629 if (!pMixedCtx->ldtr.Attr.u)
4630 u32Access = X86DESCATTR_UNUSABLE;
4631 else
4632 u32Access = pMixedCtx->ldtr.Attr.u;
4633
4634 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4635 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4636 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4637 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4638 AssertRCReturn(rc, rc);
4639
4640 /* Validate. */
4641 if (!(u32Access & X86DESCATTR_UNUSABLE))
4642 {
4643 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4644 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4645 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4646 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4647 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4648 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4649 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4650 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4651 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4652 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4653 }
4654
4655 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4656 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4657 }
4658
4659 /*
4660 * Guest IDTR.
4661 */
4662 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4663 {
4664 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4665 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4666 AssertRCReturn(rc, rc);
4667
4668 /* Validate. */
4669 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4670
4671 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4672 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4673 }
4674
4675 return VINF_SUCCESS;
4676}
4677
4678
4679/**
4680 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4681 * areas.
4682 *
4683 * These MSRs will automatically be loaded to the host CPU on every successful
4684 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4685 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4686 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4687 *
4688 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4689 *
4690 * @returns VBox status code.
4691 * @param pVCpu The cross context virtual CPU structure.
4692 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4693 * out-of-sync. Make sure to update the required fields
4694 * before using them.
4695 *
4696 * @remarks No-long-jump zone!!!
4697 */
4698static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4699{
4700 AssertPtr(pVCpu);
4701 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4702
4703 /*
4704 * MSRs that we use the auto-load/store MSR area in the VMCS.
4705 */
4706 PVM pVM = pVCpu->CTX_SUFF(pVM);
4707 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4708 {
4709 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4710#if HC_ARCH_BITS == 32
4711 if (pVM->hm.s.fAllow64BitGuests)
4712 {
4713 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4714 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4715 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4716 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4717 AssertRCReturn(rc, rc);
4718# ifdef LOG_ENABLED
4719 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4720 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4721 {
4722 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4723 pMsr->u64Value));
4724 }
4725# endif
4726 }
4727#endif
4728 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4729 }
4730
4731 /*
4732 * Guest Sysenter MSRs.
4733 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4734 * VM-exits on WRMSRs for these MSRs.
4735 */
4736 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4737 {
4738 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4739 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4740 }
4741
4742 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4743 {
4744 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4745 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4746 }
4747
4748 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4749 {
4750 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4752 }
4753
4754 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4755 {
4756 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4757 {
4758 /*
4759 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4760 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4761 */
4762 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4763 {
4764 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4765 AssertRCReturn(rc,rc);
4766 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4767 }
4768 else
4769 {
4770 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4771 NULL /* pfAddedAndUpdated */);
4772 AssertRCReturn(rc, rc);
4773
4774 /* We need to intercept reads too, see @bugref{7386#c16}. */
4775 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4776 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4777 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4778 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4779 }
4780 }
4781 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4782 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4783 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4784 }
4785
4786 return VINF_SUCCESS;
4787}
4788
4789
4790/**
4791 * Loads the guest activity state into the guest-state area in the VMCS.
4792 *
4793 * @returns VBox status code.
4794 * @param pVCpu The cross context virtual CPU structure.
4795 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4796 * out-of-sync. Make sure to update the required fields
4797 * before using them.
4798 *
4799 * @remarks No-long-jump zone!!!
4800 */
4801static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4802{
4803 NOREF(pMixedCtx);
4804 /** @todo See if we can make use of other states, e.g.
4805 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4806 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4807 {
4808 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4809 AssertRCReturn(rc, rc);
4810
4811 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4812 }
4813 return VINF_SUCCESS;
4814}
4815
4816
4817/**
4818 * Sets up the appropriate function to run guest code.
4819 *
4820 * @returns VBox status code.
4821 * @param pVCpu The cross context virtual CPU structure.
4822 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4823 * out-of-sync. Make sure to update the required fields
4824 * before using them.
4825 *
4826 * @remarks No-long-jump zone!!!
4827 */
4828static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4829{
4830 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4831 {
4832#ifndef VBOX_ENABLE_64_BITS_GUESTS
4833 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4834#endif
4835 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4836#if HC_ARCH_BITS == 32
4837 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4838 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4839 {
4840 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4841 {
4842 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4843 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4844 | HM_CHANGED_VMX_ENTRY_CTLS
4845 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4846 }
4847 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4848 }
4849#else
4850 /* 64-bit host. */
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4852#endif
4853 }
4854 else
4855 {
4856 /* Guest is not in long mode, use the 32-bit handler. */
4857#if HC_ARCH_BITS == 32
4858 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4859 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4860 {
4861 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4862 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4863 | HM_CHANGED_VMX_ENTRY_CTLS
4864 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4865 }
4866#endif
4867 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4868 }
4869 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4870 return VINF_SUCCESS;
4871}
4872
4873
4874/**
4875 * Wrapper for running the guest code in VT-x.
4876 *
4877 * @returns VBox status code, no informational status codes.
4878 * @param pVM The cross context VM structure.
4879 * @param pVCpu The cross context virtual CPU structure.
4880 * @param pCtx Pointer to the guest-CPU context.
4881 *
4882 * @remarks No-long-jump zone!!!
4883 */
4884DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4885{
4886 /*
4887 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4888 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4889 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4890 */
4891 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4892 /** @todo Add stats for resume vs launch. */
4893#ifdef VBOX_WITH_KERNEL_USING_XMM
4894 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4895#else
4896 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4897#endif
4898 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4899 return rc;
4900}
4901
4902
4903/**
4904 * Reports world-switch error and dumps some useful debug info.
4905 *
4906 * @param pVM The cross context VM structure.
4907 * @param pVCpu The cross context virtual CPU structure.
4908 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4909 * @param pCtx Pointer to the guest-CPU context.
4910 * @param pVmxTransient Pointer to the VMX transient structure (only
4911 * exitReason updated).
4912 */
4913static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4914{
4915 Assert(pVM);
4916 Assert(pVCpu);
4917 Assert(pCtx);
4918 Assert(pVmxTransient);
4919 HMVMX_ASSERT_PREEMPT_SAFE();
4920
4921 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4922 switch (rcVMRun)
4923 {
4924 case VERR_VMX_INVALID_VMXON_PTR:
4925 AssertFailed();
4926 break;
4927 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4928 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4929 {
4930 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4931 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4932 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4933 AssertRC(rc);
4934
4935 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4936 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4937 Cannot do it here as we may have been long preempted. */
4938
4939#ifdef VBOX_STRICT
4940 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4941 pVmxTransient->uExitReason));
4942 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4943 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4944 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4945 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4946 else
4947 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4948 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4949 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4950
4951 /* VMX control bits. */
4952 uint32_t u32Val;
4953 uint64_t u64Val;
4954 RTHCUINTREG uHCReg;
4955 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4956 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4957 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4958 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4959 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4960 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4961 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4962 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4963 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4964 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4965 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4966 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4967 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4968 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4969 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4970 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4971 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4972 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4987 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4988 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4989 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4990 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4991 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4992 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4993 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4994 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4995 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4996 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4997
4998 /* Guest bits. */
4999 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5000 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5001 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5002 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5003 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5004 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5007
5008 /* Host bits. */
5009 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5010 Log4(("Host CR0 %#RHr\n", uHCReg));
5011 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5012 Log4(("Host CR3 %#RHr\n", uHCReg));
5013 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5014 Log4(("Host CR4 %#RHr\n", uHCReg));
5015
5016 RTGDTR HostGdtr;
5017 PCX86DESCHC pDesc;
5018 ASMGetGDTR(&HostGdtr);
5019 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5020 Log4(("Host CS %#08x\n", u32Val));
5021 if (u32Val < HostGdtr.cbGdt)
5022 {
5023 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5024 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5025 }
5026
5027 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5028 Log4(("Host DS %#08x\n", u32Val));
5029 if (u32Val < HostGdtr.cbGdt)
5030 {
5031 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5032 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5033 }
5034
5035 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5036 Log4(("Host ES %#08x\n", u32Val));
5037 if (u32Val < HostGdtr.cbGdt)
5038 {
5039 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5040 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5041 }
5042
5043 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5044 Log4(("Host FS %#08x\n", u32Val));
5045 if (u32Val < HostGdtr.cbGdt)
5046 {
5047 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5048 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5049 }
5050
5051 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5052 Log4(("Host GS %#08x\n", u32Val));
5053 if (u32Val < HostGdtr.cbGdt)
5054 {
5055 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5056 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5057 }
5058
5059 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5060 Log4(("Host SS %#08x\n", u32Val));
5061 if (u32Val < HostGdtr.cbGdt)
5062 {
5063 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5064 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5065 }
5066
5067 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5068 Log4(("Host TR %#08x\n", u32Val));
5069 if (u32Val < HostGdtr.cbGdt)
5070 {
5071 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5072 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5073 }
5074
5075 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5076 Log4(("Host TR Base %#RHv\n", uHCReg));
5077 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5078 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5079 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5080 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5081 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5082 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5083 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5084 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5085 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5086 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5088 Log4(("Host RSP %#RHv\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5090 Log4(("Host RIP %#RHv\n", uHCReg));
5091# if HC_ARCH_BITS == 64
5092 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5093 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5094 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5095 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5096 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5097 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5098# endif
5099#endif /* VBOX_STRICT */
5100 break;
5101 }
5102
5103 default:
5104 /* Impossible */
5105 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5106 break;
5107 }
5108 NOREF(pVM); NOREF(pCtx);
5109}
5110
5111
5112#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5113#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5114# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5115#endif
5116#ifdef VBOX_STRICT
5117static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5118{
5119 switch (idxField)
5120 {
5121 case VMX_VMCS_GUEST_RIP:
5122 case VMX_VMCS_GUEST_RSP:
5123 case VMX_VMCS_GUEST_SYSENTER_EIP:
5124 case VMX_VMCS_GUEST_SYSENTER_ESP:
5125 case VMX_VMCS_GUEST_GDTR_BASE:
5126 case VMX_VMCS_GUEST_IDTR_BASE:
5127 case VMX_VMCS_GUEST_CS_BASE:
5128 case VMX_VMCS_GUEST_DS_BASE:
5129 case VMX_VMCS_GUEST_ES_BASE:
5130 case VMX_VMCS_GUEST_FS_BASE:
5131 case VMX_VMCS_GUEST_GS_BASE:
5132 case VMX_VMCS_GUEST_SS_BASE:
5133 case VMX_VMCS_GUEST_LDTR_BASE:
5134 case VMX_VMCS_GUEST_TR_BASE:
5135 case VMX_VMCS_GUEST_CR3:
5136 return true;
5137 }
5138 return false;
5139}
5140
5141static bool hmR0VmxIsValidReadField(uint32_t idxField)
5142{
5143 switch (idxField)
5144 {
5145 /* Read-only fields. */
5146 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5147 return true;
5148 }
5149 /* Remaining readable fields should also be writable. */
5150 return hmR0VmxIsValidWriteField(idxField);
5151}
5152#endif /* VBOX_STRICT */
5153
5154
5155/**
5156 * Executes the specified handler in 64-bit mode.
5157 *
5158 * @returns VBox status code (no informational status codes).
5159 * @param pVM The cross context VM structure.
5160 * @param pVCpu The cross context virtual CPU structure.
5161 * @param pCtx Pointer to the guest CPU context.
5162 * @param enmOp The operation to perform.
5163 * @param cParams Number of parameters.
5164 * @param paParam Array of 32-bit parameters.
5165 */
5166VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5167 uint32_t cParams, uint32_t *paParam)
5168{
5169 NOREF(pCtx);
5170
5171 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5172 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5173 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5174 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5175
5176#ifdef VBOX_STRICT
5177 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5178 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5179
5180 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5181 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5182#endif
5183
5184 /* Disable interrupts. */
5185 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5186
5187#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5188 RTCPUID idHostCpu = RTMpCpuId();
5189 CPUMR0SetLApic(pVCpu, idHostCpu);
5190#endif
5191
5192 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5193 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5194
5195 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5196 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5197
5198 /* Leave VMX Root Mode. */
5199 VMXDisable();
5200
5201 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5202
5203 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5204 CPUMSetHyperEIP(pVCpu, enmOp);
5205 for (int i = (int)cParams - 1; i >= 0; i--)
5206 CPUMPushHyper(pVCpu, paParam[i]);
5207
5208 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5209
5210 /* Call the switcher. */
5211 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5212 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5213
5214 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5215 /* Make sure the VMX instructions don't cause #UD faults. */
5216 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5217
5218 /* Re-enter VMX Root Mode */
5219 int rc2 = VMXEnable(HCPhysCpuPage);
5220 if (RT_FAILURE(rc2))
5221 {
5222 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5223 ASMSetFlags(fOldEFlags);
5224 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5225 return rc2;
5226 }
5227
5228 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5229 AssertRC(rc2);
5230 Assert(!(ASMGetFlags() & X86_EFL_IF));
5231 ASMSetFlags(fOldEFlags);
5232 return rc;
5233}
5234
5235
5236/**
5237 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5238 * supporting 64-bit guests.
5239 *
5240 * @returns VBox status code.
5241 * @param fResume Whether to VMLAUNCH or VMRESUME.
5242 * @param pCtx Pointer to the guest-CPU context.
5243 * @param pCache Pointer to the VMCS cache.
5244 * @param pVM The cross context VM structure.
5245 * @param pVCpu The cross context virtual CPU structure.
5246 */
5247DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5248{
5249 NOREF(fResume);
5250
5251 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5252 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5253
5254#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5255 pCache->uPos = 1;
5256 pCache->interPD = PGMGetInterPaeCR3(pVM);
5257 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5258#endif
5259
5260#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5261 pCache->TestIn.HCPhysCpuPage = 0;
5262 pCache->TestIn.HCPhysVmcs = 0;
5263 pCache->TestIn.pCache = 0;
5264 pCache->TestOut.HCPhysVmcs = 0;
5265 pCache->TestOut.pCache = 0;
5266 pCache->TestOut.pCtx = 0;
5267 pCache->TestOut.eflags = 0;
5268#else
5269 NOREF(pCache);
5270#endif
5271
5272 uint32_t aParam[10];
5273 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5274 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5275 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5276 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5277 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5278 aParam[5] = 0;
5279 aParam[6] = VM_RC_ADDR(pVM, pVM);
5280 aParam[7] = 0;
5281 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5282 aParam[9] = 0;
5283
5284#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5285 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5286 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5287#endif
5288 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5289
5290#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5291 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5292 Assert(pCtx->dr[4] == 10);
5293 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5294#endif
5295
5296#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5297 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5298 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5299 pVCpu->hm.s.vmx.HCPhysVmcs));
5300 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5301 pCache->TestOut.HCPhysVmcs));
5302 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5303 pCache->TestOut.pCache));
5304 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5305 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5306 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5307 pCache->TestOut.pCtx));
5308 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5309#endif
5310 return rc;
5311}
5312
5313
5314/**
5315 * Initialize the VMCS-Read cache.
5316 *
5317 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5318 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5319 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5320 * (those that have a 32-bit FULL & HIGH part).
5321 *
5322 * @returns VBox status code.
5323 * @param pVM The cross context VM structure.
5324 * @param pVCpu The cross context virtual CPU structure.
5325 */
5326static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5327{
5328#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5329{ \
5330 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5331 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5332 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5333 ++cReadFields; \
5334}
5335
5336 AssertPtr(pVM);
5337 AssertPtr(pVCpu);
5338 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5339 uint32_t cReadFields = 0;
5340
5341 /*
5342 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5343 * and serve to indicate exceptions to the rules.
5344 */
5345
5346 /* Guest-natural selector base fields. */
5347#if 0
5348 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5351#endif
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5354 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5364#if 0
5365 /* Unused natural width guest-state fields. */
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5368#endif
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5371
5372 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5373#if 0
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5383#endif
5384
5385 /* Natural width guest-state fields. */
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5387#if 0
5388 /* Currently unused field. */
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5390#endif
5391
5392 if (pVM->hm.s.fNestedPaging)
5393 {
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5395 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5396 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5397 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5398 }
5399 else
5400 {
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5402 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5403 }
5404
5405#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5406 return VINF_SUCCESS;
5407}
5408
5409
5410/**
5411 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5412 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5413 * darwin, running 64-bit guests).
5414 *
5415 * @returns VBox status code.
5416 * @param pVCpu The cross context virtual CPU structure.
5417 * @param idxField The VMCS field encoding.
5418 * @param u64Val 16, 32 or 64-bit value.
5419 */
5420VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5421{
5422 int rc;
5423 switch (idxField)
5424 {
5425 /*
5426 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5427 */
5428 /* 64-bit Control fields. */
5429 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5430 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5431 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5432 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5433 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5434 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5435 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5436 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5437 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5438 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5439 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5440 case VMX_VMCS64_CTRL_EPTP_FULL:
5441 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5442 /* 64-bit Guest-state fields. */
5443 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5444 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5445 case VMX_VMCS64_GUEST_PAT_FULL:
5446 case VMX_VMCS64_GUEST_EFER_FULL:
5447 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5448 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5449 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5450 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5451 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5452 /* 64-bit Host-state fields. */
5453 case VMX_VMCS64_HOST_PAT_FULL:
5454 case VMX_VMCS64_HOST_EFER_FULL:
5455 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5456 {
5457 rc = VMXWriteVmcs32(idxField, u64Val);
5458 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5459 break;
5460 }
5461
5462 /*
5463 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5464 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5465 */
5466 /* Natural-width Guest-state fields. */
5467 case VMX_VMCS_GUEST_CR3:
5468 case VMX_VMCS_GUEST_ES_BASE:
5469 case VMX_VMCS_GUEST_CS_BASE:
5470 case VMX_VMCS_GUEST_SS_BASE:
5471 case VMX_VMCS_GUEST_DS_BASE:
5472 case VMX_VMCS_GUEST_FS_BASE:
5473 case VMX_VMCS_GUEST_GS_BASE:
5474 case VMX_VMCS_GUEST_LDTR_BASE:
5475 case VMX_VMCS_GUEST_TR_BASE:
5476 case VMX_VMCS_GUEST_GDTR_BASE:
5477 case VMX_VMCS_GUEST_IDTR_BASE:
5478 case VMX_VMCS_GUEST_RSP:
5479 case VMX_VMCS_GUEST_RIP:
5480 case VMX_VMCS_GUEST_SYSENTER_ESP:
5481 case VMX_VMCS_GUEST_SYSENTER_EIP:
5482 {
5483 if (!(u64Val >> 32))
5484 {
5485 /* If this field is 64-bit, VT-x will zero out the top bits. */
5486 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5487 }
5488 else
5489 {
5490 /* Assert that only the 32->64 switcher case should ever come here. */
5491 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5492 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5493 }
5494 break;
5495 }
5496
5497 default:
5498 {
5499 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5500 rc = VERR_INVALID_PARAMETER;
5501 break;
5502 }
5503 }
5504 AssertRCReturn(rc, rc);
5505 return rc;
5506}
5507
5508
5509/**
5510 * Queue up a VMWRITE by using the VMCS write cache.
5511 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5512 *
5513 * @param pVCpu The cross context virtual CPU structure.
5514 * @param idxField The VMCS field encoding.
5515 * @param u64Val 16, 32 or 64-bit value.
5516 */
5517VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5518{
5519 AssertPtr(pVCpu);
5520 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5521
5522 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5523 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5524
5525 /* Make sure there are no duplicates. */
5526 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5527 {
5528 if (pCache->Write.aField[i] == idxField)
5529 {
5530 pCache->Write.aFieldVal[i] = u64Val;
5531 return VINF_SUCCESS;
5532 }
5533 }
5534
5535 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5536 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5537 pCache->Write.cValidEntries++;
5538 return VINF_SUCCESS;
5539}
5540#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5541
5542
5543/**
5544 * Sets up the usage of TSC-offsetting and updates the VMCS.
5545 *
5546 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5547 * VMX preemption timer.
5548 *
5549 * @returns VBox status code.
5550 * @param pVM The cross context VM structure.
5551 * @param pVCpu The cross context virtual CPU structure.
5552 *
5553 * @remarks No-long-jump zone!!!
5554 */
5555static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5556{
5557 int rc;
5558 bool fOffsettedTsc;
5559 bool fParavirtTsc;
5560 if (pVM->hm.s.vmx.fUsePreemptTimer)
5561 {
5562 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5563 &fOffsettedTsc, &fParavirtTsc);
5564
5565 /* Make sure the returned values have sane upper and lower boundaries. */
5566 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5567 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5568 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5569 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5570
5571 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5572 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5573 }
5574 else
5575 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5576
5577 /** @todo later optimize this to be done elsewhere and not before every
5578 * VM-entry. */
5579 if (fParavirtTsc)
5580 {
5581 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5582 information before every VM-entry, hence disable it for performance sake. */
5583#if 0
5584 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5585 AssertRC(rc);
5586#endif
5587 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5588 }
5589
5590 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5591 {
5592 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5593 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5594
5595 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5596 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5597 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5598 }
5599 else
5600 {
5601 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5602 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5603 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5604 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5605 }
5606}
5607
5608
5609/**
5610 * Determines if an exception is a contributory exception.
5611 *
5612 * Contributory exceptions are ones which can cause double-faults unless the
5613 * original exception was a benign exception. Page-fault is intentionally not
5614 * included here as it's a conditional contributory exception.
5615 *
5616 * @returns true if the exception is contributory, false otherwise.
5617 * @param uVector The exception vector.
5618 */
5619DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5620{
5621 switch (uVector)
5622 {
5623 case X86_XCPT_GP:
5624 case X86_XCPT_SS:
5625 case X86_XCPT_NP:
5626 case X86_XCPT_TS:
5627 case X86_XCPT_DE:
5628 return true;
5629 default:
5630 break;
5631 }
5632 return false;
5633}
5634
5635
5636/**
5637 * Sets an event as a pending event to be injected into the guest.
5638 *
5639 * @param pVCpu The cross context virtual CPU structure.
5640 * @param u32IntInfo The VM-entry interruption-information field.
5641 * @param cbInstr The VM-entry instruction length in bytes (for software
5642 * interrupts, exceptions and privileged software
5643 * exceptions).
5644 * @param u32ErrCode The VM-entry exception error code.
5645 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5646 * page-fault.
5647 *
5648 * @remarks Statistics counter assumes this is a guest event being injected or
5649 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5650 * always incremented.
5651 */
5652DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5653 RTGCUINTPTR GCPtrFaultAddress)
5654{
5655 Assert(!pVCpu->hm.s.Event.fPending);
5656 pVCpu->hm.s.Event.fPending = true;
5657 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5658 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5659 pVCpu->hm.s.Event.cbInstr = cbInstr;
5660 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5661
5662 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5663}
5664
5665
5666/**
5667 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5668 *
5669 * @param pVCpu The cross context virtual CPU structure.
5670 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5671 * out-of-sync. Make sure to update the required fields
5672 * before using them.
5673 */
5674DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675{
5676 NOREF(pMixedCtx);
5677 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5678 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5679 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5680 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5681}
5682
5683
5684/**
5685 * Handle a condition that occurred while delivering an event through the guest
5686 * IDT.
5687 *
5688 * @returns Strict VBox status code (i.e. informational status codes too).
5689 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5690 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5691 * to continue execution of the guest which will delivery the \#DF.
5692 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5693 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5694 *
5695 * @param pVCpu The cross context virtual CPU structure.
5696 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5697 * out-of-sync. Make sure to update the required fields
5698 * before using them.
5699 * @param pVmxTransient Pointer to the VMX transient structure.
5700 *
5701 * @remarks No-long-jump zone!!!
5702 */
5703static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5704{
5705 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5706
5707 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5708 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5709
5710 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5711 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5712 {
5713 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5714 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5715
5716 typedef enum
5717 {
5718 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5719 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5720 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5721 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5722 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5723 } VMXREFLECTXCPT;
5724
5725 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5726 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5727 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5728 {
5729 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5730 {
5731 enmReflect = VMXREFLECTXCPT_XCPT;
5732#ifdef VBOX_STRICT
5733 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5734 && uExitVector == X86_XCPT_PF)
5735 {
5736 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5737 }
5738#endif
5739 if ( uExitVector == X86_XCPT_PF
5740 && uIdtVector == X86_XCPT_PF)
5741 {
5742 pVmxTransient->fVectoringDoublePF = true;
5743 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5744 }
5745 else if ( uExitVector == X86_XCPT_AC
5746 && uIdtVector == X86_XCPT_AC)
5747 {
5748 enmReflect = VMXREFLECTXCPT_HANG;
5749 Log4(("IDT: Nested #AC - Bad guest\n"));
5750 }
5751 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5752 && hmR0VmxIsContributoryXcpt(uExitVector)
5753 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5754 || uIdtVector == X86_XCPT_PF))
5755 {
5756 enmReflect = VMXREFLECTXCPT_DF;
5757 }
5758 else if (uIdtVector == X86_XCPT_DF)
5759 enmReflect = VMXREFLECTXCPT_TF;
5760 }
5761 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5762 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5763 {
5764 /*
5765 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5766 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5767 */
5768 enmReflect = VMXREFLECTXCPT_XCPT;
5769
5770 if (uExitVector == X86_XCPT_PF)
5771 {
5772 pVmxTransient->fVectoringPF = true;
5773 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5774 }
5775 }
5776 }
5777 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5778 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5779 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5780 {
5781 /*
5782 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5783 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5784 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5785 */
5786 enmReflect = VMXREFLECTXCPT_XCPT;
5787 }
5788
5789 /*
5790 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5791 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5792 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5793 *
5794 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5795 */
5796 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5797 && enmReflect == VMXREFLECTXCPT_XCPT
5798 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5799 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5800 {
5801 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5802 }
5803
5804 switch (enmReflect)
5805 {
5806 case VMXREFLECTXCPT_XCPT:
5807 {
5808 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5809 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5810 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5811
5812 uint32_t u32ErrCode = 0;
5813 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5814 {
5815 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5816 AssertRCReturn(rc2, rc2);
5817 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5818 }
5819
5820 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5821 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5822 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5823 rcStrict = VINF_SUCCESS;
5824 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5825 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5826
5827 break;
5828 }
5829
5830 case VMXREFLECTXCPT_DF:
5831 {
5832 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5833 rcStrict = VINF_HM_DOUBLE_FAULT;
5834 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5835 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5836
5837 break;
5838 }
5839
5840 case VMXREFLECTXCPT_TF:
5841 {
5842 rcStrict = VINF_EM_RESET;
5843 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5844 uExitVector));
5845 break;
5846 }
5847
5848 case VMXREFLECTXCPT_HANG:
5849 {
5850 rcStrict = VERR_EM_GUEST_CPU_HANG;
5851 break;
5852 }
5853
5854 default:
5855 Assert(rcStrict == VINF_SUCCESS);
5856 break;
5857 }
5858 }
5859 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5860 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5861 && uExitVector != X86_XCPT_DF
5862 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5863 {
5864 /*
5865 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5866 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5867 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5868 */
5869 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5870 {
5871 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5872 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5873 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5874 }
5875 }
5876
5877 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5878 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5879 return rcStrict;
5880}
5881
5882
5883/**
5884 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5885 *
5886 * @returns VBox status code.
5887 * @param pVCpu The cross context virtual CPU structure.
5888 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5889 * out-of-sync. Make sure to update the required fields
5890 * before using them.
5891 *
5892 * @remarks No-long-jump zone!!!
5893 */
5894static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5895{
5896 NOREF(pMixedCtx);
5897
5898 /*
5899 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5900 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5901 */
5902 VMMRZCallRing3Disable(pVCpu);
5903 HM_DISABLE_PREEMPT();
5904
5905 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5906 {
5907 uint32_t uVal = 0;
5908 uint32_t uShadow = 0;
5909 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5910 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5911 AssertRCReturn(rc, rc);
5912
5913 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5914 CPUMSetGuestCR0(pVCpu, uVal);
5915 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5916 }
5917
5918 HM_RESTORE_PREEMPT();
5919 VMMRZCallRing3Enable(pVCpu);
5920 return VINF_SUCCESS;
5921}
5922
5923
5924/**
5925 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5926 *
5927 * @returns VBox status code.
5928 * @param pVCpu The cross context virtual CPU structure.
5929 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5930 * out-of-sync. Make sure to update the required fields
5931 * before using them.
5932 *
5933 * @remarks No-long-jump zone!!!
5934 */
5935static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5936{
5937 NOREF(pMixedCtx);
5938
5939 int rc = VINF_SUCCESS;
5940 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5941 {
5942 uint32_t uVal = 0;
5943 uint32_t uShadow = 0;
5944 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5945 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5946 AssertRCReturn(rc, rc);
5947
5948 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5949 CPUMSetGuestCR4(pVCpu, uVal);
5950 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5951 }
5952 return rc;
5953}
5954
5955
5956/**
5957 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5958 *
5959 * @returns VBox status code.
5960 * @param pVCpu The cross context virtual CPU structure.
5961 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5962 * out-of-sync. Make sure to update the required fields
5963 * before using them.
5964 *
5965 * @remarks No-long-jump zone!!!
5966 */
5967static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5968{
5969 int rc = VINF_SUCCESS;
5970 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5971 {
5972 uint64_t u64Val = 0;
5973 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5974 AssertRCReturn(rc, rc);
5975
5976 pMixedCtx->rip = u64Val;
5977 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5978 }
5979 return rc;
5980}
5981
5982
5983/**
5984 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5985 *
5986 * @returns VBox status code.
5987 * @param pVCpu The cross context virtual CPU structure.
5988 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5989 * out-of-sync. Make sure to update the required fields
5990 * before using them.
5991 *
5992 * @remarks No-long-jump zone!!!
5993 */
5994static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5995{
5996 int rc = VINF_SUCCESS;
5997 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5998 {
5999 uint64_t u64Val = 0;
6000 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6001 AssertRCReturn(rc, rc);
6002
6003 pMixedCtx->rsp = u64Val;
6004 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6005 }
6006 return rc;
6007}
6008
6009
6010/**
6011 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu The cross context virtual CPU structure.
6015 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6016 * out-of-sync. Make sure to update the required fields
6017 * before using them.
6018 *
6019 * @remarks No-long-jump zone!!!
6020 */
6021static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6022{
6023 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6024 {
6025 uint32_t uVal = 0;
6026 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6027 AssertRCReturn(rc, rc);
6028
6029 pMixedCtx->eflags.u32 = uVal;
6030 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6031 {
6032 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6033 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6034
6035 pMixedCtx->eflags.Bits.u1VM = 0;
6036 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6037 }
6038
6039 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6040 }
6041 return VINF_SUCCESS;
6042}
6043
6044
6045/**
6046 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6047 * guest-CPU context.
6048 */
6049DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6050{
6051 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6052 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6053 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6054 return rc;
6055}
6056
6057
6058/**
6059 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6060 * from the guest-state area in the VMCS.
6061 *
6062 * @param pVCpu The cross context virtual CPU structure.
6063 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6064 * out-of-sync. Make sure to update the required fields
6065 * before using them.
6066 *
6067 * @remarks No-long-jump zone!!!
6068 */
6069static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6070{
6071 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6072 {
6073 uint32_t uIntrState = 0;
6074 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6075 AssertRC(rc);
6076
6077 if (!uIntrState)
6078 {
6079 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6080 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6081
6082 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6083 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6084 }
6085 else
6086 {
6087 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6088 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6089 {
6090 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6091 AssertRC(rc);
6092 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6093 AssertRC(rc);
6094
6095 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6096 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6097 }
6098 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6099 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6100
6101 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6102 {
6103 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6104 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6105 }
6106 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6107 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6108 }
6109
6110 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6111 }
6112}
6113
6114
6115/**
6116 * Saves the guest's activity state.
6117 *
6118 * @returns VBox status code.
6119 * @param pVCpu The cross context virtual CPU structure.
6120 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6121 * out-of-sync. Make sure to update the required fields
6122 * before using them.
6123 *
6124 * @remarks No-long-jump zone!!!
6125 */
6126static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6127{
6128 NOREF(pMixedCtx);
6129 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6130 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6131 return VINF_SUCCESS;
6132}
6133
6134
6135/**
6136 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6137 * the current VMCS into the guest-CPU context.
6138 *
6139 * @returns VBox status code.
6140 * @param pVCpu The cross context virtual CPU structure.
6141 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6142 * out-of-sync. Make sure to update the required fields
6143 * before using them.
6144 *
6145 * @remarks No-long-jump zone!!!
6146 */
6147static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6148{
6149 int rc = VINF_SUCCESS;
6150 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6151 {
6152 uint32_t u32Val = 0;
6153 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6154 pMixedCtx->SysEnter.cs = u32Val;
6155 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6156 }
6157
6158 uint64_t u64Val = 0;
6159 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6160 {
6161 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6162 pMixedCtx->SysEnter.eip = u64Val;
6163 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6164 }
6165 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6166 {
6167 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6168 pMixedCtx->SysEnter.esp = u64Val;
6169 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6170 }
6171 return rc;
6172}
6173
6174
6175/**
6176 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6177 * the CPU back into the guest-CPU context.
6178 *
6179 * @returns VBox status code.
6180 * @param pVCpu The cross context virtual CPU structure.
6181 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6182 * out-of-sync. Make sure to update the required fields
6183 * before using them.
6184 *
6185 * @remarks No-long-jump zone!!!
6186 */
6187static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6188{
6189 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6190 VMMRZCallRing3Disable(pVCpu);
6191 HM_DISABLE_PREEMPT();
6192
6193 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6194 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6195 {
6196 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6197 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6198 }
6199
6200 HM_RESTORE_PREEMPT();
6201 VMMRZCallRing3Enable(pVCpu);
6202
6203 return VINF_SUCCESS;
6204}
6205
6206
6207/**
6208 * Saves the auto load/store'd guest MSRs from the current VMCS into
6209 * the guest-CPU context.
6210 *
6211 * @returns VBox status code.
6212 * @param pVCpu The cross context virtual CPU structure.
6213 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6214 * out-of-sync. Make sure to update the required fields
6215 * before using them.
6216 *
6217 * @remarks No-long-jump zone!!!
6218 */
6219static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6220{
6221 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6222 return VINF_SUCCESS;
6223
6224 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6225 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6226 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6227 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6228 {
6229 switch (pMsr->u32Msr)
6230 {
6231 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6232 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6233 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6234 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6235 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6236 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6237 break;
6238
6239 default:
6240 {
6241 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6242 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6243 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6244 }
6245 }
6246 }
6247
6248 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6249 return VINF_SUCCESS;
6250}
6251
6252
6253/**
6254 * Saves the guest control registers from the current VMCS into the guest-CPU
6255 * context.
6256 *
6257 * @returns VBox status code.
6258 * @param pVCpu The cross context virtual CPU structure.
6259 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6260 * out-of-sync. Make sure to update the required fields
6261 * before using them.
6262 *
6263 * @remarks No-long-jump zone!!!
6264 */
6265static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6266{
6267 /* Guest CR0. Guest FPU. */
6268 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6269 AssertRCReturn(rc, rc);
6270
6271 /* Guest CR4. */
6272 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6273 AssertRCReturn(rc, rc);
6274
6275 /* Guest CR2 - updated always during the world-switch or in #PF. */
6276 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6277 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6278 {
6279 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6280 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6281
6282 PVM pVM = pVCpu->CTX_SUFF(pVM);
6283 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6284 || ( pVM->hm.s.fNestedPaging
6285 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6286 {
6287 uint64_t u64Val = 0;
6288 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6289 if (pMixedCtx->cr3 != u64Val)
6290 {
6291 CPUMSetGuestCR3(pVCpu, u64Val);
6292 if (VMMRZCallRing3IsEnabled(pVCpu))
6293 {
6294 PGMUpdateCR3(pVCpu, u64Val);
6295 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6296 }
6297 else
6298 {
6299 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6300 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6301 }
6302 }
6303
6304 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6305 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6306 {
6307 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6308 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6309 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6310 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6311 AssertRCReturn(rc, rc);
6312
6313 if (VMMRZCallRing3IsEnabled(pVCpu))
6314 {
6315 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6316 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6317 }
6318 else
6319 {
6320 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6321 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6322 }
6323 }
6324 }
6325
6326 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6327 }
6328
6329 /*
6330 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6331 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6332 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6333 *
6334 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6335 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6336 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6337 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6338 *
6339 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6340 */
6341 if (VMMRZCallRing3IsEnabled(pVCpu))
6342 {
6343 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6344 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6345
6346 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6347 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6348
6349 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6350 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6351 }
6352
6353 return rc;
6354}
6355
6356
6357/**
6358 * Reads a guest segment register from the current VMCS into the guest-CPU
6359 * context.
6360 *
6361 * @returns VBox status code.
6362 * @param pVCpu The cross context virtual CPU structure.
6363 * @param idxSel Index of the selector in the VMCS.
6364 * @param idxLimit Index of the segment limit in the VMCS.
6365 * @param idxBase Index of the segment base in the VMCS.
6366 * @param idxAccess Index of the access rights of the segment in the VMCS.
6367 * @param pSelReg Pointer to the segment selector.
6368 *
6369 * @remarks No-long-jump zone!!!
6370 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6371 * macro as that takes care of whether to read from the VMCS cache or
6372 * not.
6373 */
6374DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6375 PCPUMSELREG pSelReg)
6376{
6377 NOREF(pVCpu);
6378
6379 uint32_t u32Val = 0;
6380 int rc = VMXReadVmcs32(idxSel, &u32Val);
6381 AssertRCReturn(rc, rc);
6382 pSelReg->Sel = (uint16_t)u32Val;
6383 pSelReg->ValidSel = (uint16_t)u32Val;
6384 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6385
6386 rc = VMXReadVmcs32(idxLimit, &u32Val);
6387 AssertRCReturn(rc, rc);
6388 pSelReg->u32Limit = u32Val;
6389
6390 uint64_t u64Val = 0;
6391 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6392 AssertRCReturn(rc, rc);
6393 pSelReg->u64Base = u64Val;
6394
6395 rc = VMXReadVmcs32(idxAccess, &u32Val);
6396 AssertRCReturn(rc, rc);
6397 pSelReg->Attr.u = u32Val;
6398
6399 /*
6400 * If VT-x marks the segment as unusable, most other bits remain undefined:
6401 * - For CS the L, D and G bits have meaning.
6402 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6403 * - For the remaining data segments no bits are defined.
6404 *
6405 * The present bit and the unusable bit has been observed to be set at the
6406 * same time (the selector was supposed to be invalid as we started executing
6407 * a V8086 interrupt in ring-0).
6408 *
6409 * What should be important for the rest of the VBox code, is that the P bit is
6410 * cleared. Some of the other VBox code recognizes the unusable bit, but
6411 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6412 * safe side here, we'll strip off P and other bits we don't care about. If
6413 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6414 *
6415 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6416 */
6417 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6418 {
6419 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6420
6421 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6422 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6423 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6424
6425 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6426#ifdef DEBUG_bird
6427 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6428 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6429 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6430#endif
6431 }
6432 return VINF_SUCCESS;
6433}
6434
6435
6436#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6437# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6438 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6439 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6440#else
6441# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6442 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6443 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6444#endif
6445
6446
6447/**
6448 * Saves the guest segment registers from the current VMCS into the guest-CPU
6449 * context.
6450 *
6451 * @returns VBox status code.
6452 * @param pVCpu The cross context virtual CPU structure.
6453 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6454 * out-of-sync. Make sure to update the required fields
6455 * before using them.
6456 *
6457 * @remarks No-long-jump zone!!!
6458 */
6459static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6460{
6461 /* Guest segment registers. */
6462 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6463 {
6464 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6465 AssertRCReturn(rc, rc);
6466
6467 rc = VMXLOCAL_READ_SEG(CS, cs);
6468 rc |= VMXLOCAL_READ_SEG(SS, ss);
6469 rc |= VMXLOCAL_READ_SEG(DS, ds);
6470 rc |= VMXLOCAL_READ_SEG(ES, es);
6471 rc |= VMXLOCAL_READ_SEG(FS, fs);
6472 rc |= VMXLOCAL_READ_SEG(GS, gs);
6473 AssertRCReturn(rc, rc);
6474
6475 /* Restore segment attributes for real-on-v86 mode hack. */
6476 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6477 {
6478 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6479 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6480 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6481 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6482 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6483 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6484 }
6485 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6486 }
6487
6488 return VINF_SUCCESS;
6489}
6490
6491
6492/**
6493 * Saves the guest descriptor table registers and task register from the current
6494 * VMCS into the guest-CPU context.
6495 *
6496 * @returns VBox status code.
6497 * @param pVCpu The cross context virtual CPU structure.
6498 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6499 * out-of-sync. Make sure to update the required fields
6500 * before using them.
6501 *
6502 * @remarks No-long-jump zone!!!
6503 */
6504static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6505{
6506 int rc = VINF_SUCCESS;
6507
6508 /* Guest LDTR. */
6509 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6510 {
6511 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6512 AssertRCReturn(rc, rc);
6513 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6514 }
6515
6516 /* Guest GDTR. */
6517 uint64_t u64Val = 0;
6518 uint32_t u32Val = 0;
6519 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6520 {
6521 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6522 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6523 pMixedCtx->gdtr.pGdt = u64Val;
6524 pMixedCtx->gdtr.cbGdt = u32Val;
6525 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6526 }
6527
6528 /* Guest IDTR. */
6529 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6530 {
6531 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6532 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6533 pMixedCtx->idtr.pIdt = u64Val;
6534 pMixedCtx->idtr.cbIdt = u32Val;
6535 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6536 }
6537
6538 /* Guest TR. */
6539 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6540 {
6541 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6542 AssertRCReturn(rc, rc);
6543
6544 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6545 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6546 {
6547 rc = VMXLOCAL_READ_SEG(TR, tr);
6548 AssertRCReturn(rc, rc);
6549 }
6550 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6551 }
6552 return rc;
6553}
6554
6555#undef VMXLOCAL_READ_SEG
6556
6557
6558/**
6559 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6560 * context.
6561 *
6562 * @returns VBox status code.
6563 * @param pVCpu The cross context virtual CPU structure.
6564 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6565 * out-of-sync. Make sure to update the required fields
6566 * before using them.
6567 *
6568 * @remarks No-long-jump zone!!!
6569 */
6570static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6571{
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6573 {
6574 if (!pVCpu->hm.s.fUsingHyperDR7)
6575 {
6576 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6577 uint32_t u32Val;
6578 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6579 pMixedCtx->dr[7] = u32Val;
6580 }
6581
6582 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6583 }
6584 return VINF_SUCCESS;
6585}
6586
6587
6588/**
6589 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6590 *
6591 * @returns VBox status code.
6592 * @param pVCpu The cross context virtual CPU structure.
6593 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6594 * out-of-sync. Make sure to update the required fields
6595 * before using them.
6596 *
6597 * @remarks No-long-jump zone!!!
6598 */
6599static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6600{
6601 NOREF(pMixedCtx);
6602
6603 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6604 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6605 return VINF_SUCCESS;
6606}
6607
6608
6609/**
6610 * Saves the entire guest state from the currently active VMCS into the
6611 * guest-CPU context.
6612 *
6613 * This essentially VMREADs all guest-data.
6614 *
6615 * @returns VBox status code.
6616 * @param pVCpu The cross context virtual CPU structure.
6617 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6618 * out-of-sync. Make sure to update the required fields
6619 * before using them.
6620 */
6621static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6622{
6623 Assert(pVCpu);
6624 Assert(pMixedCtx);
6625
6626 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6627 return VINF_SUCCESS;
6628
6629 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6630 again on the ring-3 callback path, there is no real need to. */
6631 if (VMMRZCallRing3IsEnabled(pVCpu))
6632 VMMR0LogFlushDisable(pVCpu);
6633 else
6634 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6635 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6636
6637 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6638 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6639
6640 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6641 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6642
6643 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6644 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6645
6646 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6647 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6648
6649 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6650 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6651
6652 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6653 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6654
6655 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6656 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6657
6658 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6659 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6660
6661 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6662 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6663
6664 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6665 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6666
6667 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6668 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6669 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6670
6671 if (VMMRZCallRing3IsEnabled(pVCpu))
6672 VMMR0LogFlushEnable(pVCpu);
6673
6674 return VINF_SUCCESS;
6675}
6676
6677
6678/**
6679 * Saves basic guest registers needed for IEM instruction execution.
6680 *
6681 * @returns VBox status code (OR-able).
6682 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6683 * @param pMixedCtx Pointer to the CPU context of the guest.
6684 * @param fMemory Whether the instruction being executed operates on
6685 * memory or not. Only CR0 is synced up if clear.
6686 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6687 */
6688static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6689{
6690 /*
6691 * We assume all general purpose registers other than RSP are available.
6692 *
6693 * RIP is a must, as it will be incremented or otherwise changed.
6694 *
6695 * RFLAGS are always required to figure the CPL.
6696 *
6697 * RSP isn't always required, however it's a GPR, so frequently required.
6698 *
6699 * SS and CS are the only segment register needed if IEM doesn't do memory
6700 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6701 *
6702 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6703 * be required for memory accesses.
6704 *
6705 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6706 */
6707 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6708 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6709 if (fNeedRsp)
6710 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6711 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6712 if (!fMemory)
6713 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6714 else
6715 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6716 AssertRCReturn(rc, rc);
6717 return rc;
6718}
6719
6720
6721/**
6722 * Ensures that we've got a complete basic guest-context.
6723 *
6724 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6725 * is for the interpreter.
6726 *
6727 * @returns VBox status code.
6728 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6729 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6730 * needing to be synced in.
6731 * @thread EMT(pVCpu)
6732 */
6733VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6734{
6735 /* Note! Since this is only applicable to VT-x, the implementation is placed
6736 in the VT-x part of the sources instead of the generic stuff. */
6737 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6738 {
6739 /* For now, imply that the caller might change everything too. */
6740 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6741 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6742 }
6743 return VINF_SUCCESS;
6744}
6745
6746
6747/**
6748 * Check per-VM and per-VCPU force flag actions that require us to go back to
6749 * ring-3 for one reason or another.
6750 *
6751 * @returns Strict VBox status code (i.e. informational status codes too)
6752 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6753 * ring-3.
6754 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6755 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6756 * interrupts)
6757 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6758 * all EMTs to be in ring-3.
6759 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6760 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6761 * to the EM loop.
6762 *
6763 * @param pVM The cross context VM structure.
6764 * @param pVCpu The cross context virtual CPU structure.
6765 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6766 * out-of-sync. Make sure to update the required fields
6767 * before using them.
6768 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6769 */
6770static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6771{
6772 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6773
6774 /*
6775 * Anything pending? Should be more likely than not if we're doing a good job.
6776 */
6777 if ( !fStepping
6778 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6779 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6780 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6781 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6782 return VINF_SUCCESS;
6783
6784 /* We need the control registers now, make sure the guest-CPU context is updated. */
6785 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6786 AssertRCReturn(rc3, rc3);
6787
6788 /* Pending HM CR3 sync. */
6789 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6790 {
6791 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6792 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6793 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6794 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6795 }
6796
6797 /* Pending HM PAE PDPEs. */
6798 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6799 {
6800 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6801 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6802 }
6803
6804 /* Pending PGM C3 sync. */
6805 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6806 {
6807 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6808 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6809 if (rcStrict2 != VINF_SUCCESS)
6810 {
6811 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6812 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6813 return rcStrict2;
6814 }
6815 }
6816
6817 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6818 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6819 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6820 {
6821 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6822 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6823 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6824 return rc2;
6825 }
6826
6827 /* Pending VM request packets, such as hardware interrupts. */
6828 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6829 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6830 {
6831 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6832 return VINF_EM_PENDING_REQUEST;
6833 }
6834
6835 /* Pending PGM pool flushes. */
6836 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6837 {
6838 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6839 return VINF_PGM_POOL_FLUSH_PENDING;
6840 }
6841
6842 /* Pending DMA requests. */
6843 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6844 {
6845 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6846 return VINF_EM_RAW_TO_R3;
6847 }
6848
6849 return VINF_SUCCESS;
6850}
6851
6852
6853/**
6854 * Converts any TRPM trap into a pending HM event. This is typically used when
6855 * entering from ring-3 (not longjmp returns).
6856 *
6857 * @param pVCpu The cross context virtual CPU structure.
6858 */
6859static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6860{
6861 Assert(TRPMHasTrap(pVCpu));
6862 Assert(!pVCpu->hm.s.Event.fPending);
6863
6864 uint8_t uVector;
6865 TRPMEVENT enmTrpmEvent;
6866 RTGCUINT uErrCode;
6867 RTGCUINTPTR GCPtrFaultAddress;
6868 uint8_t cbInstr;
6869
6870 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6871 AssertRC(rc);
6872
6873 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6874 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6875 if (enmTrpmEvent == TRPM_TRAP)
6876 {
6877 switch (uVector)
6878 {
6879 case X86_XCPT_NMI:
6880 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6881 break;
6882
6883 case X86_XCPT_BP:
6884 case X86_XCPT_OF:
6885 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6886 break;
6887
6888 case X86_XCPT_PF:
6889 case X86_XCPT_DF:
6890 case X86_XCPT_TS:
6891 case X86_XCPT_NP:
6892 case X86_XCPT_SS:
6893 case X86_XCPT_GP:
6894 case X86_XCPT_AC:
6895 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6896 /* no break! */
6897 default:
6898 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6899 break;
6900 }
6901 }
6902 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6903 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6904 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6905 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6906 else
6907 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6908
6909 rc = TRPMResetTrap(pVCpu);
6910 AssertRC(rc);
6911 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6912 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6913
6914 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6915 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6916}
6917
6918
6919/**
6920 * Converts the pending HM event into a TRPM trap.
6921 *
6922 * @param pVCpu The cross context virtual CPU structure.
6923 */
6924static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6925{
6926 Assert(pVCpu->hm.s.Event.fPending);
6927
6928 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6929 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6930 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6931 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6932
6933 /* If a trap was already pending, we did something wrong! */
6934 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6935
6936 TRPMEVENT enmTrapType;
6937 switch (uVectorType)
6938 {
6939 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6940 enmTrapType = TRPM_HARDWARE_INT;
6941 break;
6942
6943 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6944 enmTrapType = TRPM_SOFTWARE_INT;
6945 break;
6946
6947 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6948 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6949 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6950 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6951 enmTrapType = TRPM_TRAP;
6952 break;
6953
6954 default:
6955 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6956 enmTrapType = TRPM_32BIT_HACK;
6957 break;
6958 }
6959
6960 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6961
6962 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6963 AssertRC(rc);
6964
6965 if (fErrorCodeValid)
6966 TRPMSetErrorCode(pVCpu, uErrorCode);
6967
6968 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6969 && uVector == X86_XCPT_PF)
6970 {
6971 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6972 }
6973 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6974 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6975 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6976 {
6977 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6978 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6979 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6980 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6981 }
6982
6983 /* Clear any pending events from the VMCS. */
6984 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6985 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6986
6987 /* We're now done converting the pending event. */
6988 pVCpu->hm.s.Event.fPending = false;
6989}
6990
6991
6992/**
6993 * Does the necessary state syncing before returning to ring-3 for any reason
6994 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6995 *
6996 * @returns VBox status code.
6997 * @param pVM The cross context VM structure.
6998 * @param pVCpu The cross context virtual CPU structure.
6999 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7000 * be out-of-sync. Make sure to update the required
7001 * fields before using them.
7002 * @param fSaveGuestState Whether to save the guest state or not.
7003 *
7004 * @remarks No-long-jmp zone!!!
7005 */
7006static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7007{
7008 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7009 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7010
7011 RTCPUID idCpu = RTMpCpuId();
7012 Log4Func(("HostCpuId=%u\n", idCpu));
7013
7014 /*
7015 * !!! IMPORTANT !!!
7016 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7017 */
7018
7019 /* Save the guest state if necessary. */
7020 if ( fSaveGuestState
7021 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7022 {
7023 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7024 AssertRCReturn(rc, rc);
7025 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7026 }
7027
7028 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7029 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7030 {
7031 if (fSaveGuestState)
7032 {
7033 /* We shouldn't reload CR0 without saving it first. */
7034 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7035 AssertRCReturn(rc, rc);
7036 }
7037 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7038 }
7039
7040 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7041#ifdef VBOX_STRICT
7042 if (CPUMIsHyperDebugStateActive(pVCpu))
7043 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7044#endif
7045 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7046 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7047 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7048 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7049
7050#if HC_ARCH_BITS == 64
7051 /* Restore host-state bits that VT-x only restores partially. */
7052 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7053 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7054 {
7055 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7056 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7057 }
7058 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7059#endif
7060
7061 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7062 if (pVCpu->hm.s.vmx.fLazyMsrs)
7063 {
7064 /* We shouldn't reload the guest MSRs without saving it first. */
7065 if (!fSaveGuestState)
7066 {
7067 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7068 AssertRCReturn(rc, rc);
7069 }
7070 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7071 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7072 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7073 }
7074
7075 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7076 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7077
7078 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7079 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7080 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7081 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7082 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7083 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7084 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7085 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7086
7087 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7088
7089 /** @todo This partially defeats the purpose of having preemption hooks.
7090 * The problem is, deregistering the hooks should be moved to a place that
7091 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7092 * context.
7093 */
7094 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7095 {
7096 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7097 AssertRCReturn(rc, rc);
7098
7099 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7100 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7101 }
7102 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7103 NOREF(idCpu);
7104
7105 return VINF_SUCCESS;
7106}
7107
7108
7109/**
7110 * Leaves the VT-x session.
7111 *
7112 * @returns VBox status code.
7113 * @param pVM The cross context VM structure.
7114 * @param pVCpu The cross context virtual CPU structure.
7115 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7116 * out-of-sync. Make sure to update the required fields
7117 * before using them.
7118 *
7119 * @remarks No-long-jmp zone!!!
7120 */
7121DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7122{
7123 HM_DISABLE_PREEMPT();
7124 HMVMX_ASSERT_CPU_SAFE();
7125 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7126 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7127
7128 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7129 and done this from the VMXR0ThreadCtxCallback(). */
7130 if (!pVCpu->hm.s.fLeaveDone)
7131 {
7132 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7133 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7134 pVCpu->hm.s.fLeaveDone = true;
7135 }
7136 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7137
7138 /*
7139 * !!! IMPORTANT !!!
7140 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7141 */
7142
7143 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7144 /** @todo Deregistering here means we need to VMCLEAR always
7145 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7146 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7147 VMMR0ThreadCtxHookDisable(pVCpu);
7148
7149 /* Leave HM context. This takes care of local init (term). */
7150 int rc = HMR0LeaveCpu(pVCpu);
7151
7152 HM_RESTORE_PREEMPT();
7153 return rc;
7154}
7155
7156
7157/**
7158 * Does the necessary state syncing before doing a longjmp to ring-3.
7159 *
7160 * @returns VBox status code.
7161 * @param pVM The cross context VM structure.
7162 * @param pVCpu The cross context virtual CPU structure.
7163 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7164 * out-of-sync. Make sure to update the required fields
7165 * before using them.
7166 *
7167 * @remarks No-long-jmp zone!!!
7168 */
7169DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7170{
7171 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7172}
7173
7174
7175/**
7176 * Take necessary actions before going back to ring-3.
7177 *
7178 * An action requires us to go back to ring-3. This function does the necessary
7179 * steps before we can safely return to ring-3. This is not the same as longjmps
7180 * to ring-3, this is voluntary and prepares the guest so it may continue
7181 * executing outside HM (recompiler/IEM).
7182 *
7183 * @returns VBox status code.
7184 * @param pVM The cross context VM structure.
7185 * @param pVCpu The cross context virtual CPU structure.
7186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7187 * out-of-sync. Make sure to update the required fields
7188 * before using them.
7189 * @param rcExit The reason for exiting to ring-3. Can be
7190 * VINF_VMM_UNKNOWN_RING3_CALL.
7191 */
7192static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7193{
7194 Assert(pVM);
7195 Assert(pVCpu);
7196 Assert(pMixedCtx);
7197 HMVMX_ASSERT_PREEMPT_SAFE();
7198
7199 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7200 {
7201 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7202 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7203 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7204 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7205 }
7206
7207 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7208 VMMRZCallRing3Disable(pVCpu);
7209 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7210
7211 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7212 if (pVCpu->hm.s.Event.fPending)
7213 {
7214 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7215 Assert(!pVCpu->hm.s.Event.fPending);
7216 }
7217
7218 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7219 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7220
7221 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7222 and if we're injecting an event we should have a TRPM trap pending. */
7223 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7224#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7225 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7226#endif
7227
7228 /* Save guest state and restore host state bits. */
7229 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7230 AssertRCReturn(rc, rc);
7231 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7232 /* Thread-context hooks are unregistered at this point!!! */
7233
7234 /* Sync recompiler state. */
7235 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7236 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7237 | CPUM_CHANGED_LDTR
7238 | CPUM_CHANGED_GDTR
7239 | CPUM_CHANGED_IDTR
7240 | CPUM_CHANGED_TR
7241 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7242 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7243 if ( pVM->hm.s.fNestedPaging
7244 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7245 {
7246 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7247 }
7248
7249 Assert(!pVCpu->hm.s.fClearTrapFlag);
7250
7251 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7252 if (rcExit != VINF_EM_RAW_INTERRUPT)
7253 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7254
7255 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7256
7257 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7258 VMMRZCallRing3RemoveNotification(pVCpu);
7259 VMMRZCallRing3Enable(pVCpu);
7260
7261 return rc;
7262}
7263
7264
7265/**
7266 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7267 * longjump to ring-3 and possibly get preempted.
7268 *
7269 * @returns VBox status code.
7270 * @param pVCpu The cross context virtual CPU structure.
7271 * @param enmOperation The operation causing the ring-3 longjump.
7272 * @param pvUser Opaque pointer to the guest-CPU context. The data
7273 * may be out-of-sync. Make sure to update the required
7274 * fields before using them.
7275 */
7276static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7277{
7278 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7279 {
7280 /*
7281 * !!! IMPORTANT !!!
7282 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7283 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7284 */
7285 VMMRZCallRing3RemoveNotification(pVCpu);
7286 VMMRZCallRing3Disable(pVCpu);
7287 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7288 RTThreadPreemptDisable(&PreemptState);
7289
7290 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7291 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7292
7293#if HC_ARCH_BITS == 64
7294 /* Restore host-state bits that VT-x only restores partially. */
7295 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7296 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7297 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7298 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7299#endif
7300 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7301 if (pVCpu->hm.s.vmx.fLazyMsrs)
7302 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7303
7304 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7305 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7306 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7307 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7308 {
7309 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7310 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7311 }
7312
7313 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7314 VMMR0ThreadCtxHookDisable(pVCpu);
7315 HMR0LeaveCpu(pVCpu);
7316 RTThreadPreemptRestore(&PreemptState);
7317 return VINF_SUCCESS;
7318 }
7319
7320 Assert(pVCpu);
7321 Assert(pvUser);
7322 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7323 HMVMX_ASSERT_PREEMPT_SAFE();
7324
7325 VMMRZCallRing3Disable(pVCpu);
7326 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7327
7328 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7329 enmOperation));
7330
7331 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7332 AssertRCReturn(rc, rc);
7333
7334 VMMRZCallRing3Enable(pVCpu);
7335 return VINF_SUCCESS;
7336}
7337
7338
7339/**
7340 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7341 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7342 *
7343 * @param pVCpu The cross context virtual CPU structure.
7344 */
7345DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7346{
7347 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7348 {
7349 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7350 {
7351 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7352 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7353 AssertRC(rc);
7354 Log4(("Setup interrupt-window exiting\n"));
7355 }
7356 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7357}
7358
7359
7360/**
7361 * Clears the interrupt-window exiting control in the VMCS.
7362 *
7363 * @param pVCpu The cross context virtual CPU structure.
7364 */
7365DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7366{
7367 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7368 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7369 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7370 AssertRC(rc);
7371 Log4(("Cleared interrupt-window exiting\n"));
7372}
7373
7374
7375/**
7376 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7377 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7378 *
7379 * @param pVCpu The cross context virtual CPU structure.
7380 */
7381DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7382{
7383 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7384 {
7385 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7386 {
7387 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7388 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7389 AssertRC(rc);
7390 Log4(("Setup NMI-window exiting\n"));
7391 }
7392 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7393}
7394
7395
7396/**
7397 * Clears the NMI-window exiting control in the VMCS.
7398 *
7399 * @param pVCpu The cross context virtual CPU structure.
7400 */
7401DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7402{
7403 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7404 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7405 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7406 AssertRC(rc);
7407 Log4(("Cleared NMI-window exiting\n"));
7408}
7409
7410
7411/**
7412 * Evaluates the event to be delivered to the guest and sets it as the pending
7413 * event.
7414 *
7415 * @returns The VT-x guest-interruptibility state.
7416 * @param pVCpu The cross context virtual CPU structure.
7417 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7418 * out-of-sync. Make sure to update the required fields
7419 * before using them.
7420 */
7421static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7422{
7423 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7424 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7425 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7426 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7427 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7428
7429 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7430 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7431 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7432 Assert(!TRPMHasTrap(pVCpu));
7433
7434#ifdef VBOX_WITH_NEW_APIC
7435 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7436 APICUpdatePendingInterrupts(pVCpu);
7437#endif
7438
7439 /*
7440 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7441 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7442 */
7443 /** @todo SMI. SMIs take priority over NMIs. */
7444 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7445 {
7446 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7447 if ( !pVCpu->hm.s.Event.fPending
7448 && !fBlockNmi
7449 && !fBlockSti
7450 && !fBlockMovSS)
7451 {
7452 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7453 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7454 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7455
7456 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7457 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7458 }
7459 else
7460 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7461 }
7462 /*
7463 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7464 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7465 */
7466 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7467 && !pVCpu->hm.s.fSingleInstruction)
7468 {
7469 Assert(!DBGFIsStepping(pVCpu));
7470 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7471 AssertRC(rc);
7472 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7473 if ( !pVCpu->hm.s.Event.fPending
7474 && !fBlockInt
7475 && !fBlockSti
7476 && !fBlockMovSS)
7477 {
7478 uint8_t u8Interrupt;
7479 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7480 if (RT_SUCCESS(rc))
7481 {
7482 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7483 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7484 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485
7486 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7487 }
7488 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7489 {
7490 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7491 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7492 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7493 }
7494 else
7495 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7496 }
7497 else
7498 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7499 }
7500
7501 return uIntrState;
7502}
7503
7504
7505/**
7506 * Sets a pending-debug exception to be delivered to the guest if the guest is
7507 * single-stepping in the VMCS.
7508 *
7509 * @param pVCpu The cross context virtual CPU structure.
7510 */
7511DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7512{
7513 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7514 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7515 AssertRC(rc);
7516}
7517
7518
7519/**
7520 * Injects any pending events into the guest if the guest is in a state to
7521 * receive them.
7522 *
7523 * @returns Strict VBox status code (i.e. informational status codes too).
7524 * @param pVCpu The cross context virtual CPU structure.
7525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7526 * out-of-sync. Make sure to update the required fields
7527 * before using them.
7528 * @param uIntrState The VT-x guest-interruptibility state.
7529 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7530 * return VINF_EM_DBG_STEPPED if the event was
7531 * dispatched directly.
7532 */
7533static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7534{
7535 HMVMX_ASSERT_PREEMPT_SAFE();
7536 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7537
7538 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7539 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7540
7541 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7542 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7543 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7544 Assert(!TRPMHasTrap(pVCpu));
7545
7546 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7547 if (pVCpu->hm.s.Event.fPending)
7548 {
7549 /*
7550 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7551 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7552 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7553 *
7554 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7555 */
7556 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7557#ifdef VBOX_STRICT
7558 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7559 {
7560 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7561 Assert(!fBlockInt);
7562 Assert(!fBlockSti);
7563 Assert(!fBlockMovSS);
7564 }
7565 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7566 {
7567 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7568 Assert(!fBlockSti);
7569 Assert(!fBlockMovSS);
7570 Assert(!fBlockNmi);
7571 }
7572#endif
7573 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7574 (uint8_t)uIntType));
7575 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7576 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7577 fStepping, &uIntrState);
7578 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7579
7580 /* Update the interruptibility-state as it could have been changed by
7581 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7582 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7583 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7584
7585 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7586 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7587 else
7588 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7589 }
7590
7591 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7592 if ( fBlockSti
7593 || fBlockMovSS)
7594 {
7595 if (!pVCpu->hm.s.fSingleInstruction)
7596 {
7597 /*
7598 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7599 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7600 * See Intel spec. 27.3.4 "Saving Non-Register State".
7601 */
7602 Assert(!DBGFIsStepping(pVCpu));
7603 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7604 AssertRCReturn(rc2, rc2);
7605 if (pMixedCtx->eflags.Bits.u1TF)
7606 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7607 }
7608 else if (pMixedCtx->eflags.Bits.u1TF)
7609 {
7610 /*
7611 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7612 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7613 */
7614 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7615 uIntrState = 0;
7616 }
7617 }
7618
7619 /*
7620 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7621 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7622 */
7623 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7624 AssertRC(rc2);
7625
7626 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7627 NOREF(fBlockMovSS); NOREF(fBlockSti);
7628 return rcStrict;
7629}
7630
7631
7632/**
7633 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7634 *
7635 * @param pVCpu The cross context virtual CPU structure.
7636 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7637 * out-of-sync. Make sure to update the required fields
7638 * before using them.
7639 */
7640DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7641{
7642 NOREF(pMixedCtx);
7643 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7644 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7645}
7646
7647
7648/**
7649 * Injects a double-fault (\#DF) exception into the VM.
7650 *
7651 * @returns Strict VBox status code (i.e. informational status codes too).
7652 * @param pVCpu The cross context virtual CPU structure.
7653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7654 * out-of-sync. Make sure to update the required fields
7655 * before using them.
7656 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7657 * and should return VINF_EM_DBG_STEPPED if the event
7658 * is injected directly (register modified by us, not
7659 * by hardware on VM-entry).
7660 * @param puIntrState Pointer to the current guest interruptibility-state.
7661 * This interruptibility-state will be updated if
7662 * necessary. This cannot not be NULL.
7663 */
7664DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7665{
7666 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7667 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7668 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7669 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7670 fStepping, puIntrState);
7671}
7672
7673
7674/**
7675 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7676 *
7677 * @param pVCpu The cross context virtual CPU structure.
7678 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7679 * out-of-sync. Make sure to update the required fields
7680 * before using them.
7681 */
7682DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7683{
7684 NOREF(pMixedCtx);
7685 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7686 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7687 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7688}
7689
7690
7691/**
7692 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7693 *
7694 * @param pVCpu The cross context virtual CPU structure.
7695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7696 * out-of-sync. Make sure to update the required fields
7697 * before using them.
7698 * @param cbInstr The value of RIP that is to be pushed on the guest
7699 * stack.
7700 */
7701DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7702{
7703 NOREF(pMixedCtx);
7704 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7705 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7706 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7707}
7708
7709
7710/**
7711 * Injects a general-protection (\#GP) fault into the VM.
7712 *
7713 * @returns Strict VBox status code (i.e. informational status codes too).
7714 * @param pVCpu The cross context virtual CPU structure.
7715 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7716 * out-of-sync. Make sure to update the required fields
7717 * before using them.
7718 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7719 * mode, i.e. in real-mode it's not valid).
7720 * @param u32ErrorCode The error code associated with the \#GP.
7721 * @param fStepping Whether we're running in
7722 * hmR0VmxRunGuestCodeStep() and should return
7723 * VINF_EM_DBG_STEPPED if the event is injected
7724 * directly (register modified by us, not by
7725 * hardware on VM-entry).
7726 * @param puIntrState Pointer to the current guest interruptibility-state.
7727 * This interruptibility-state will be updated if
7728 * necessary. This cannot not be NULL.
7729 */
7730DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7731 bool fStepping, uint32_t *puIntrState)
7732{
7733 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7734 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7735 if (fErrorCodeValid)
7736 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7737 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7738 fStepping, puIntrState);
7739}
7740
7741
7742/**
7743 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7744 * VM.
7745 *
7746 * @param pVCpu The cross context virtual CPU structure.
7747 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7748 * out-of-sync. Make sure to update the required fields
7749 * before using them.
7750 * @param u32ErrorCode The error code associated with the \#GP.
7751 */
7752DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7753{
7754 NOREF(pMixedCtx);
7755 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7756 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7757 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7758 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7759}
7760
7761
7762/**
7763 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7764 *
7765 * @param pVCpu The cross context virtual CPU structure.
7766 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7767 * out-of-sync. Make sure to update the required fields
7768 * before using them.
7769 * @param uVector The software interrupt vector number.
7770 * @param cbInstr The value of RIP that is to be pushed on the guest
7771 * stack.
7772 */
7773DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7774{
7775 NOREF(pMixedCtx);
7776 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7777 if ( uVector == X86_XCPT_BP
7778 || uVector == X86_XCPT_OF)
7779 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7780 else
7781 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7782 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7783}
7784
7785
7786/**
7787 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7788 * stack.
7789 *
7790 * @returns Strict VBox status code (i.e. informational status codes too).
7791 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7792 * @param pVM The cross context VM structure.
7793 * @param pMixedCtx Pointer to the guest-CPU context.
7794 * @param uValue The value to push to the guest stack.
7795 */
7796DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7797{
7798 /*
7799 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7800 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7801 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7802 */
7803 if (pMixedCtx->sp == 1)
7804 return VINF_EM_RESET;
7805 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7806 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7807 AssertRC(rc);
7808 return rc;
7809}
7810
7811
7812/**
7813 * Injects an event into the guest upon VM-entry by updating the relevant fields
7814 * in the VM-entry area in the VMCS.
7815 *
7816 * @returns Strict VBox status code (i.e. informational status codes too).
7817 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7818 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7819 *
7820 * @param pVCpu The cross context virtual CPU structure.
7821 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7822 * be out-of-sync. Make sure to update the required
7823 * fields before using them.
7824 * @param u64IntInfo The VM-entry interruption-information field.
7825 * @param cbInstr The VM-entry instruction length in bytes (for
7826 * software interrupts, exceptions and privileged
7827 * software exceptions).
7828 * @param u32ErrCode The VM-entry exception error code.
7829 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7830 * @param puIntrState Pointer to the current guest interruptibility-state.
7831 * This interruptibility-state will be updated if
7832 * necessary. This cannot not be NULL.
7833 * @param fStepping Whether we're running in
7834 * hmR0VmxRunGuestCodeStep() and should return
7835 * VINF_EM_DBG_STEPPED if the event is injected
7836 * directly (register modified by us, not by
7837 * hardware on VM-entry).
7838 *
7839 * @remarks Requires CR0!
7840 * @remarks No-long-jump zone!!!
7841 */
7842static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7843 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7844 uint32_t *puIntrState)
7845{
7846 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7847 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7848 Assert(puIntrState);
7849 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7850
7851 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7852 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7853
7854#ifdef VBOX_STRICT
7855 /* Validate the error-code-valid bit for hardware exceptions. */
7856 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7857 {
7858 switch (uVector)
7859 {
7860 case X86_XCPT_PF:
7861 case X86_XCPT_DF:
7862 case X86_XCPT_TS:
7863 case X86_XCPT_NP:
7864 case X86_XCPT_SS:
7865 case X86_XCPT_GP:
7866 case X86_XCPT_AC:
7867 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7868 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7869 /* fallthru */
7870 default:
7871 break;
7872 }
7873 }
7874#endif
7875
7876 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7877 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7878 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7879
7880 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7881
7882 /* We require CR0 to check if the guest is in real-mode. */
7883 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7884 AssertRCReturn(rc, rc);
7885
7886 /*
7887 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7888 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7889 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7890 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7891 */
7892 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7893 {
7894 PVM pVM = pVCpu->CTX_SUFF(pVM);
7895 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7896 {
7897 Assert(PDMVmmDevHeapIsEnabled(pVM));
7898 Assert(pVM->hm.s.vmx.pRealModeTSS);
7899
7900 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7901 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7902 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7903 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7904 AssertRCReturn(rc, rc);
7905 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7906
7907 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7908 size_t const cbIdtEntry = sizeof(X86IDTR16);
7909 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7910 {
7911 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7912 if (uVector == X86_XCPT_DF)
7913 return VINF_EM_RESET;
7914
7915 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7916 if (uVector == X86_XCPT_GP)
7917 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7918
7919 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7920 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7921 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7922 fStepping, puIntrState);
7923 }
7924
7925 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7926 uint16_t uGuestIp = pMixedCtx->ip;
7927 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7928 {
7929 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7930 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7931 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7932 }
7933 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7934 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7935
7936 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7937 X86IDTR16 IdtEntry;
7938 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7939 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7940 AssertRCReturn(rc, rc);
7941
7942 /* Construct the stack frame for the interrupt/exception handler. */
7943 VBOXSTRICTRC rcStrict;
7944 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7945 if (rcStrict == VINF_SUCCESS)
7946 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7947 if (rcStrict == VINF_SUCCESS)
7948 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7949
7950 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7951 if (rcStrict == VINF_SUCCESS)
7952 {
7953 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7954 pMixedCtx->rip = IdtEntry.offSel;
7955 pMixedCtx->cs.Sel = IdtEntry.uSel;
7956 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7957 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7958 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7959 && uVector == X86_XCPT_PF)
7960 pMixedCtx->cr2 = GCPtrFaultAddress;
7961
7962 /* If any other guest-state bits are changed here, make sure to update
7963 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7964 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7965 | HM_CHANGED_GUEST_RIP
7966 | HM_CHANGED_GUEST_RFLAGS
7967 | HM_CHANGED_GUEST_RSP);
7968
7969 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7970 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7971 {
7972 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7973 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7974 Log4(("Clearing inhibition due to STI.\n"));
7975 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7976 }
7977 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7978 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7979
7980 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7981 it, if we are returning to ring-3 before executing guest code. */
7982 pVCpu->hm.s.Event.fPending = false;
7983
7984 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
7985 if (fStepping)
7986 rcStrict = VINF_EM_DBG_STEPPED;
7987 }
7988 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7989 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7990 return rcStrict;
7991 }
7992
7993 /*
7994 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7995 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7996 */
7997 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7998 }
7999
8000 /* Validate. */
8001 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8002 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8003 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8004
8005 /* Inject. */
8006 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8007 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8008 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8009 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8010
8011 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8012 && uVector == X86_XCPT_PF)
8013 pMixedCtx->cr2 = GCPtrFaultAddress;
8014
8015 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8016 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8017
8018 AssertRCReturn(rc, rc);
8019 return VINF_SUCCESS;
8020}
8021
8022
8023/**
8024 * Clears the interrupt-window exiting control in the VMCS and if necessary
8025 * clears the current event in the VMCS as well.
8026 *
8027 * @returns VBox status code.
8028 * @param pVCpu The cross context virtual CPU structure.
8029 *
8030 * @remarks Use this function only to clear events that have not yet been
8031 * delivered to the guest but are injected in the VMCS!
8032 * @remarks No-long-jump zone!!!
8033 */
8034static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8035{
8036 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8037
8038 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8039 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8040
8041 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8042 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8043}
8044
8045
8046/**
8047 * Enters the VT-x session.
8048 *
8049 * @returns VBox status code.
8050 * @param pVM The cross context VM structure.
8051 * @param pVCpu The cross context virtual CPU structure.
8052 * @param pCpu Pointer to the CPU info struct.
8053 */
8054VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8055{
8056 AssertPtr(pVM);
8057 AssertPtr(pVCpu);
8058 Assert(pVM->hm.s.vmx.fSupported);
8059 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8060 NOREF(pCpu); NOREF(pVM);
8061
8062 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8063 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8064
8065#ifdef VBOX_STRICT
8066 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8067 RTCCUINTREG uHostCR4 = ASMGetCR4();
8068 if (!(uHostCR4 & X86_CR4_VMXE))
8069 {
8070 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8071 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8072 }
8073#endif
8074
8075 /*
8076 * Load the VCPU's VMCS as the current (and active) one.
8077 */
8078 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8079 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8080 if (RT_FAILURE(rc))
8081 return rc;
8082
8083 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8084 pVCpu->hm.s.fLeaveDone = false;
8085 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8086
8087 return VINF_SUCCESS;
8088}
8089
8090
8091/**
8092 * The thread-context callback (only on platforms which support it).
8093 *
8094 * @param enmEvent The thread-context event.
8095 * @param pVCpu The cross context virtual CPU structure.
8096 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8097 * @thread EMT(pVCpu)
8098 */
8099VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8100{
8101 NOREF(fGlobalInit);
8102
8103 switch (enmEvent)
8104 {
8105 case RTTHREADCTXEVENT_OUT:
8106 {
8107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8108 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8109 VMCPU_ASSERT_EMT(pVCpu);
8110
8111 PVM pVM = pVCpu->CTX_SUFF(pVM);
8112 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8113
8114 /* No longjmps (logger flushes, locks) in this fragile context. */
8115 VMMRZCallRing3Disable(pVCpu);
8116 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8117
8118 /*
8119 * Restore host-state (FPU, debug etc.)
8120 */
8121 if (!pVCpu->hm.s.fLeaveDone)
8122 {
8123 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8124 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8125 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8126 pVCpu->hm.s.fLeaveDone = true;
8127 }
8128
8129 /* Leave HM context, takes care of local init (term). */
8130 int rc = HMR0LeaveCpu(pVCpu);
8131 AssertRC(rc); NOREF(rc);
8132
8133 /* Restore longjmp state. */
8134 VMMRZCallRing3Enable(pVCpu);
8135 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8136 break;
8137 }
8138
8139 case RTTHREADCTXEVENT_IN:
8140 {
8141 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8142 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8143 VMCPU_ASSERT_EMT(pVCpu);
8144
8145 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8146 VMMRZCallRing3Disable(pVCpu);
8147 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8148
8149 /* Initialize the bare minimum state required for HM. This takes care of
8150 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8151 int rc = HMR0EnterCpu(pVCpu);
8152 AssertRC(rc);
8153 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8154
8155 /* Load the active VMCS as the current one. */
8156 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8157 {
8158 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8159 AssertRC(rc); NOREF(rc);
8160 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8161 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8162 }
8163 pVCpu->hm.s.fLeaveDone = false;
8164
8165 /* Restore longjmp state. */
8166 VMMRZCallRing3Enable(pVCpu);
8167 break;
8168 }
8169
8170 default:
8171 break;
8172 }
8173}
8174
8175
8176/**
8177 * Saves the host state in the VMCS host-state.
8178 * Sets up the VM-exit MSR-load area.
8179 *
8180 * The CPU state will be loaded from these fields on every successful VM-exit.
8181 *
8182 * @returns VBox status code.
8183 * @param pVM The cross context VM structure.
8184 * @param pVCpu The cross context virtual CPU structure.
8185 *
8186 * @remarks No-long-jump zone!!!
8187 */
8188static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8189{
8190 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8191
8192 int rc = VINF_SUCCESS;
8193 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8194 {
8195 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8196 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8197
8198 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8199 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8200
8201 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8202 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8203
8204 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8205 }
8206 return rc;
8207}
8208
8209
8210/**
8211 * Saves the host state in the VMCS host-state.
8212 *
8213 * @returns VBox status code.
8214 * @param pVM The cross context VM structure.
8215 * @param pVCpu The cross context virtual CPU structure.
8216 *
8217 * @remarks No-long-jump zone!!!
8218 */
8219VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8220{
8221 AssertPtr(pVM);
8222 AssertPtr(pVCpu);
8223
8224 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8225
8226 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8227 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8228 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8229 return hmR0VmxSaveHostState(pVM, pVCpu);
8230}
8231
8232
8233/**
8234 * Loads the guest state into the VMCS guest-state area.
8235 *
8236 * The will typically be done before VM-entry when the guest-CPU state and the
8237 * VMCS state may potentially be out of sync.
8238 *
8239 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8240 * VM-entry controls.
8241 * Sets up the appropriate VMX non-root function to execute guest code based on
8242 * the guest CPU mode.
8243 *
8244 * @returns VBox strict status code.
8245 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8246 * without unrestricted guest access and the VMMDev is not presently
8247 * mapped (e.g. EFI32).
8248 *
8249 * @param pVM The cross context VM structure.
8250 * @param pVCpu The cross context virtual CPU structure.
8251 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8252 * out-of-sync. Make sure to update the required fields
8253 * before using them.
8254 *
8255 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8256 * caller disables then again on successfull return. Confusing.)
8257 */
8258static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8259{
8260 AssertPtr(pVM);
8261 AssertPtr(pVCpu);
8262 AssertPtr(pMixedCtx);
8263 HMVMX_ASSERT_PREEMPT_SAFE();
8264
8265 VMMRZCallRing3Disable(pVCpu);
8266 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8267
8268 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8269
8270 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8271
8272 /* Determine real-on-v86 mode. */
8273 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8274 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8275 && CPUMIsGuestInRealModeEx(pMixedCtx))
8276 {
8277 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8278 }
8279
8280 /*
8281 * Load the guest-state into the VMCS.
8282 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8283 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8284 */
8285 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8286 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8287
8288 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8289 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8290 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8291
8292 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8293 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8294 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8295
8296 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8297 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8298
8299 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8300 if (rcStrict == VINF_SUCCESS)
8301 { /* likely */ }
8302 else
8303 {
8304 VMMRZCallRing3Enable(pVCpu);
8305 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8306 return rcStrict;
8307 }
8308
8309 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8310 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8311 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8312
8313 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8314 determine we don't have to swap EFER after all. */
8315 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8316 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8317
8318 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8319 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8320
8321 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8322 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8323
8324 /*
8325 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8326 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8327 */
8328 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8329 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8330
8331 /* Clear any unused and reserved bits. */
8332 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8333
8334 VMMRZCallRing3Enable(pVCpu);
8335
8336 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8337 return rc;
8338}
8339
8340
8341/**
8342 * Loads the state shared between the host and guest into the VMCS.
8343 *
8344 * @param pVM The cross context VM structure.
8345 * @param pVCpu The cross context virtual CPU structure.
8346 * @param pCtx Pointer to the guest-CPU context.
8347 *
8348 * @remarks No-long-jump zone!!!
8349 */
8350static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8351{
8352 NOREF(pVM);
8353
8354 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8355 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8356
8357 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8358 {
8359 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8360 AssertRC(rc);
8361 }
8362
8363 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8364 {
8365 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8366 AssertRC(rc);
8367
8368 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8369 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8370 {
8371 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8372 AssertRC(rc);
8373 }
8374 }
8375
8376 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8377 {
8378 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8379 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8380 }
8381
8382 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8383 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8384 {
8385 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8386 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8387 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8388 AssertRC(rc);
8389 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8390 }
8391
8392 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8393 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8394}
8395
8396
8397/**
8398 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8399 *
8400 * @returns Strict VBox status code (i.e. informational status codes too).
8401 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8402 * without unrestricted guest access and the VMMDev is not presently
8403 * mapped (e.g. EFI32).
8404 *
8405 * @param pVM The cross context VM structure.
8406 * @param pVCpu The cross context virtual CPU structure.
8407 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8408 * out-of-sync. Make sure to update the required fields
8409 * before using them.
8410 */
8411static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8412{
8413 HMVMX_ASSERT_PREEMPT_SAFE();
8414
8415 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8416#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8417 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8418#endif
8419
8420 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8421 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8422 {
8423 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8424 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8425 { /* likely */}
8426 else
8427 {
8428 AssertLogRelMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8429 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8430 }
8431 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8432 }
8433 else if (HMCPU_CF_VALUE(pVCpu))
8434 {
8435 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8436 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8437 { /* likely */}
8438 else
8439 {
8440 AssertLogRelMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8441 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8442 return rcStrict;
8443 }
8444 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8445 }
8446
8447 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8448 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8449 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8450 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8451 return rcStrict;
8452}
8453
8454
8455/**
8456 * Does the preparations before executing guest code in VT-x.
8457 *
8458 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8459 * recompiler/IEM. We must be cautious what we do here regarding committing
8460 * guest-state information into the VMCS assuming we assuredly execute the
8461 * guest in VT-x mode.
8462 *
8463 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8464 * the common-state (TRPM/forceflags), we must undo those changes so that the
8465 * recompiler/IEM can (and should) use them when it resumes guest execution.
8466 * Otherwise such operations must be done when we can no longer exit to ring-3.
8467 *
8468 * @returns Strict VBox status code (i.e. informational status codes too).
8469 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8470 * have been disabled.
8471 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8472 * double-fault into the guest.
8473 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8474 * dispatched directly.
8475 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8476 *
8477 * @param pVM The cross context VM structure.
8478 * @param pVCpu The cross context virtual CPU structure.
8479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8480 * out-of-sync. Make sure to update the required fields
8481 * before using them.
8482 * @param pVmxTransient Pointer to the VMX transient structure.
8483 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8484 * us ignore some of the reasons for returning to
8485 * ring-3, and return VINF_EM_DBG_STEPPED if event
8486 * dispatching took place.
8487 */
8488static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8489{
8490 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8491
8492#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8493 PGMRZDynMapFlushAutoSet(pVCpu);
8494#endif
8495
8496 /* Check force flag actions that might require us to go back to ring-3. */
8497 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8498 if (rcStrict == VINF_SUCCESS)
8499 { /* FFs doesn't get set all the time. */ }
8500 else
8501 return rcStrict;
8502
8503#ifndef IEM_VERIFICATION_MODE_FULL
8504 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8505 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8506 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8507 {
8508 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8509 RTGCPHYS GCPhysApicBase;
8510 GCPhysApicBase = pMixedCtx->msrApicBase;
8511 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8512
8513 /* Unalias any existing mapping. */
8514 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8515 AssertRCReturn(rc, rc);
8516
8517 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8518 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8519 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8520 AssertRCReturn(rc, rc);
8521
8522 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8523 }
8524#endif /* !IEM_VERIFICATION_MODE_FULL */
8525
8526 if (TRPMHasTrap(pVCpu))
8527 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8528 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8529
8530 /*
8531 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8532 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8533 */
8534 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8535 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8536 { /* likely */ }
8537 else
8538 {
8539 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8540 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8541 return rcStrict;
8542 }
8543
8544 /*
8545 * Load the guest state bits, we can handle longjmps/getting preempted here.
8546 *
8547 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8548 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8549 * Hence, this needs to be done -after- injection of events.
8550 */
8551 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8552 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8553 { /* likely */ }
8554 else
8555 return rcStrict;
8556
8557 /*
8558 * No longjmps to ring-3 from this point on!!!
8559 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8560 * This also disables flushing of the R0-logger instance (if any).
8561 */
8562 VMMRZCallRing3Disable(pVCpu);
8563
8564 /*
8565 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8566 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8567 *
8568 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8569 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8570 *
8571 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8572 * executing guest code.
8573 */
8574 pVmxTransient->fEFlags = ASMIntDisableFlags();
8575
8576 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8577 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8578 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8579 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8580 {
8581 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8582 {
8583 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8584 pVCpu->hm.s.Event.fPending = false;
8585
8586 return VINF_SUCCESS;
8587 }
8588
8589 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8590 rcStrict = VINF_EM_RAW_INTERRUPT;
8591 }
8592 else
8593 {
8594 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8595 rcStrict = VINF_EM_RAW_TO_R3;
8596 }
8597
8598 ASMSetFlags(pVmxTransient->fEFlags);
8599 VMMRZCallRing3Enable(pVCpu);
8600
8601 return rcStrict;
8602}
8603
8604
8605/**
8606 * Prepares to run guest code in VT-x and we've committed to doing so. This
8607 * means there is no backing out to ring-3 or anywhere else at this
8608 * point.
8609 *
8610 * @param pVM The cross context VM structure.
8611 * @param pVCpu The cross context virtual CPU structure.
8612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8613 * out-of-sync. Make sure to update the required fields
8614 * before using them.
8615 * @param pVmxTransient Pointer to the VMX transient structure.
8616 *
8617 * @remarks Called with preemption disabled.
8618 * @remarks No-long-jump zone!!!
8619 */
8620static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8621{
8622 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8623 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8624 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8625
8626 /*
8627 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8628 */
8629 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8630 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8631
8632#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8633 if (!CPUMIsGuestFPUStateActive(pVCpu))
8634 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8635 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8636 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8637#endif
8638
8639 if ( pVCpu->hm.s.fPreloadGuestFpu
8640 && !CPUMIsGuestFPUStateActive(pVCpu))
8641 {
8642 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8643 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8644 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8645 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8646 }
8647
8648 /*
8649 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8650 */
8651 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8652 && pVCpu->hm.s.vmx.cMsrs > 0)
8653 {
8654 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8655 }
8656
8657 /*
8658 * Load the host state bits as we may've been preempted (only happens when
8659 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8660 */
8661 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8662 * any effect to the host state needing to be saved? */
8663 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8664 {
8665 /* This ASSUMES that pfnStartVM has been set up already. */
8666 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8667 AssertRC(rc);
8668 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8669 }
8670 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8671
8672 /*
8673 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8674 */
8675 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8676 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8677 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8678
8679 /* Store status of the shared guest-host state at the time of VM-entry. */
8680#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8681 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8682 {
8683 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8684 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8685 }
8686 else
8687#endif
8688 {
8689 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8690 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8691 }
8692 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8693
8694 /*
8695 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8696 */
8697 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8698 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8699
8700 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8701 RTCPUID idCurrentCpu = pCpu->idCpu;
8702 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8703 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8704 {
8705 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8706 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8707 }
8708
8709 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8710 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8711 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8712 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8713
8714 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8715
8716 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8717 to start executing. */
8718
8719 /*
8720 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8721 */
8722 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8723 {
8724 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8725 {
8726 bool fMsrUpdated;
8727 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8728 AssertRC(rc2);
8729 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8730
8731 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8732 &fMsrUpdated);
8733 AssertRC(rc2);
8734 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8735
8736 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8737 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8738 }
8739 else
8740 {
8741 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8742 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8743 }
8744 }
8745
8746#ifdef VBOX_STRICT
8747 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8748 hmR0VmxCheckHostEferMsr(pVCpu);
8749 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8750#endif
8751#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8752 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8753 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8754 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8755#endif
8756}
8757
8758
8759/**
8760 * Performs some essential restoration of state after running guest code in
8761 * VT-x.
8762 *
8763 * @param pVM The cross context VM structure.
8764 * @param pVCpu The cross context virtual CPU structure.
8765 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8766 * out-of-sync. Make sure to update the required fields
8767 * before using them.
8768 * @param pVmxTransient Pointer to the VMX transient structure.
8769 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8770 *
8771 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8772 *
8773 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8774 * unconditionally when it is safe to do so.
8775 */
8776static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8777{
8778 NOREF(pVM);
8779
8780 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8781
8782 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8783 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8784 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8785 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8786 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8787 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8788
8789 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8790 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8791
8792 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8793 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8794 Assert(!ASMIntAreEnabled());
8795 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8796
8797#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8798 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8799 {
8800 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8801 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8802 }
8803#endif
8804
8805#if HC_ARCH_BITS == 64
8806 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8807#endif
8808 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8809#ifdef VBOX_STRICT
8810 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8811#endif
8812 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8813 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8814
8815 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8816 uint32_t uExitReason;
8817 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8818 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8819 AssertRC(rc);
8820 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8821 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8822
8823 /* Update the VM-exit history array. */
8824 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8825
8826 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8827 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8828 {
8829 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8830 pVmxTransient->fVMEntryFailed));
8831 return;
8832 }
8833
8834 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8835 {
8836 /** @todo We can optimize this by only syncing with our force-flags when
8837 * really needed and keeping the VMCS state as it is for most
8838 * VM-exits. */
8839 /* Update the guest interruptibility-state from the VMCS. */
8840 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8841
8842#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8843 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8844 AssertRC(rc);
8845#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8846 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8847 AssertRC(rc);
8848#endif
8849
8850 /*
8851 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8852 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8853 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8854 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8855 */
8856 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8857 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8858 {
8859 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8860 AssertRC(rc);
8861 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8862 }
8863 }
8864}
8865
8866
8867/**
8868 * Runs the guest code using VT-x the normal way.
8869 *
8870 * @returns VBox status code.
8871 * @param pVM The cross context VM structure.
8872 * @param pVCpu The cross context virtual CPU structure.
8873 * @param pCtx Pointer to the guest-CPU context.
8874 *
8875 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8876 */
8877static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8878{
8879 VMXTRANSIENT VmxTransient;
8880 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8881 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8882 uint32_t cLoops = 0;
8883
8884 for (;; cLoops++)
8885 {
8886 Assert(!HMR0SuspendPending());
8887 HMVMX_ASSERT_CPU_SAFE();
8888
8889 /* Preparatory work for running guest code, this may force us to return
8890 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8891 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8892 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8893 if (rcStrict != VINF_SUCCESS)
8894 break;
8895
8896 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8897 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8898 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8899
8900 /* Restore any residual host-state and save any bits shared between host
8901 and guest into the guest-CPU state. Re-enables interrupts! */
8902 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8903
8904 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8905 if (RT_SUCCESS(rcRun))
8906 { /* very likely */ }
8907 else
8908 {
8909 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8910 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8911 return rcRun;
8912 }
8913
8914 /* Profile the VM-exit. */
8915 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8917 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8918 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8919 HMVMX_START_EXIT_DISPATCH_PROF();
8920
8921 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8922
8923 /* Handle the VM-exit. */
8924#ifdef HMVMX_USE_FUNCTION_TABLE
8925 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8926#else
8927 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8928#endif
8929 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8930 if (rcStrict == VINF_SUCCESS)
8931 {
8932 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8933 continue; /* likely */
8934 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8935 rcStrict = VINF_EM_RAW_INTERRUPT;
8936 }
8937 break;
8938 }
8939
8940 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8941 return rcStrict;
8942}
8943
8944
8945
8946/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8947 * probes.
8948 *
8949 * The following few functions and associated structure contains the bloat
8950 * necessary for providing detailed debug events and dtrace probes as well as
8951 * reliable host side single stepping. This works on the principle of
8952 * "subclassing" the normal execution loop and workers. We replace the loop
8953 * method completely and override selected helpers to add necessary adjustments
8954 * to their core operation.
8955 *
8956 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8957 * any performance for debug and analysis features.
8958 *
8959 * @{
8960 */
8961
8962typedef struct VMXRUNDBGSTATE
8963{
8964 /** The RIP we started executing at. This is for detecting that we stepped. */
8965 uint64_t uRipStart;
8966 /** The CS we started executing with. */
8967 uint16_t uCsStart;
8968
8969 /** Whether we've actually modified the 1st execution control field. */
8970 bool fModifiedProcCtls : 1;
8971 /** Whether we've actually modified the 2nd execution control field. */
8972 bool fModifiedProcCtls2 : 1;
8973 /** Whether we've actually modified the exception bitmap. */
8974 bool fModifiedXcptBitmap : 1;
8975
8976 /** We desire the modified the CR0 mask to be cleared. */
8977 bool fClearCr0Mask : 1;
8978 /** We desire the modified the CR4 mask to be cleared. */
8979 bool fClearCr4Mask : 1;
8980 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8981 uint32_t fCpe1Extra;
8982 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8983 uint32_t fCpe1Unwanted;
8984 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8985 uint32_t fCpe2Extra;
8986 /** Extra stuff we need in */
8987 uint32_t bmXcptExtra;
8988 /** The sequence number of the Dtrace provider settings the state was
8989 * configured against. */
8990 uint32_t uDtraceSettingsSeqNo;
8991 /** Exits to check (one bit per exit). */
8992 uint32_t bmExitsToCheck[3];
8993
8994 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8995 uint32_t fProcCtlsInitial;
8996 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8997 uint32_t fProcCtls2Initial;
8998 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8999 uint32_t bmXcptInitial;
9000} VMXRUNDBGSTATE;
9001AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9002typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9003
9004
9005/**
9006 * Initializes the VMXRUNDBGSTATE structure.
9007 *
9008 * @param pVCpu The cross context virtual CPU structure of the
9009 * calling EMT.
9010 * @param pCtx The CPU register context to go with @a pVCpu.
9011 * @param pDbgState The structure to initialize.
9012 */
9013DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9014{
9015 pDbgState->uRipStart = pCtx->rip;
9016 pDbgState->uCsStart = pCtx->cs.Sel;
9017
9018 pDbgState->fModifiedProcCtls = false;
9019 pDbgState->fModifiedProcCtls2 = false;
9020 pDbgState->fModifiedXcptBitmap = false;
9021 pDbgState->fClearCr0Mask = false;
9022 pDbgState->fClearCr4Mask = false;
9023 pDbgState->fCpe1Extra = 0;
9024 pDbgState->fCpe1Unwanted = 0;
9025 pDbgState->fCpe2Extra = 0;
9026 pDbgState->bmXcptExtra = 0;
9027 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9028 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9029 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9030}
9031
9032
9033/**
9034 * Updates the VMSC fields with changes requested by @a pDbgState.
9035 *
9036 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9037 * immediately before executing guest code, i.e. when interrupts are disabled.
9038 * We don't check status codes here as we cannot easily assert or return in the
9039 * latter case.
9040 *
9041 * @param pVCpu The cross context virtual CPU structure.
9042 * @param pDbgState The debug state.
9043 */
9044DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9045{
9046 /*
9047 * Ensure desired flags in VMCS control fields are set.
9048 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9049 *
9050 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9051 * there should be no stale data in pCtx at this point.
9052 */
9053 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9054 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9055 {
9056 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9057 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9058 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9059 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9060 pDbgState->fModifiedProcCtls = true;
9061 }
9062
9063 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9064 {
9065 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9066 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9067 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9068 pDbgState->fModifiedProcCtls2 = true;
9069 }
9070
9071 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9072 {
9073 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9074 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9075 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9076 pDbgState->fModifiedXcptBitmap = true;
9077 }
9078
9079 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9080 {
9081 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9082 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9083 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9084 }
9085
9086 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9087 {
9088 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9089 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9090 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9091 }
9092}
9093
9094
9095DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9096{
9097 /*
9098 * Restore exit control settings as we may not reenter this function the
9099 * next time around.
9100 */
9101 /* We reload the initial value, trigger what we can of recalculations the
9102 next time around. From the looks of things, that's all that's required atm. */
9103 if (pDbgState->fModifiedProcCtls)
9104 {
9105 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9106 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9107 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9108 AssertRCReturn(rc2, rc2);
9109 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9110 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9111 }
9112
9113 /* We're currently the only ones messing with this one, so just restore the
9114 cached value and reload the field. */
9115 if ( pDbgState->fModifiedProcCtls2
9116 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9117 {
9118 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9119 AssertRCReturn(rc2, rc2);
9120 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9121 }
9122
9123 /* If we've modified the exception bitmap, we restore it and trigger
9124 reloading and partial recalculation the next time around. */
9125 if (pDbgState->fModifiedXcptBitmap)
9126 {
9127 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9128 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9129 }
9130
9131 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9132 if (pDbgState->fClearCr0Mask)
9133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9134
9135 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9136 if (pDbgState->fClearCr4Mask)
9137 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9138
9139 return rcStrict;
9140}
9141
9142
9143/**
9144 * Configures VM-exit controls for current DBGF and DTrace settings.
9145 *
9146 * This updates @a pDbgState and the VMCS execution control fields to reflect
9147 * the necessary exits demanded by DBGF and DTrace.
9148 *
9149 * @param pVM The cross context VM structure.
9150 * @param pVCpu The cross context virtual CPU structure.
9151 * @param pCtx Pointer to the guest-CPU context.
9152 * @param pDbgState The debug state.
9153 * @param pVmxTransient Pointer to the VMX transient structure. May update
9154 * fUpdateTscOffsettingAndPreemptTimer.
9155 */
9156static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9157 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9158{
9159 /*
9160 * Take down the dtrace serial number so we can spot changes.
9161 */
9162 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9163 ASMCompilerBarrier();
9164
9165 /*
9166 * We'll rebuild most of the middle block of data members (holding the
9167 * current settings) as we go along here, so start by clearing it all.
9168 */
9169 pDbgState->bmXcptExtra = 0;
9170 pDbgState->fCpe1Extra = 0;
9171 pDbgState->fCpe1Unwanted = 0;
9172 pDbgState->fCpe2Extra = 0;
9173 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9174 pDbgState->bmExitsToCheck[i] = 0;
9175
9176 /*
9177 * Software interrupts (INT XXh) - no idea how to trigger these...
9178 */
9179 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9180 || VBOXVMM_INT_SOFTWARE_ENABLED())
9181 {
9182 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9183 }
9184
9185 /*
9186 * Exception bitmap and XCPT events+probes.
9187 */
9188 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9189 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9190 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9191
9192 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9193 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9194 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9195 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9196 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9197 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9198 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9199 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9200 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9201 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9202 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9203 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9204 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9205 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9206 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9207 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9208 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9209 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9210
9211 if (pDbgState->bmXcptExtra)
9212 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9213
9214 /*
9215 * Process events and probes for VM exits, making sure we get the wanted exits.
9216 *
9217 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9218 * So, when adding/changing/removing please don't forget to update it.
9219 *
9220 * Some of the macros are picking up local variables to save horizontal space,
9221 * (being able to see it in a table is the lesser evil here).
9222 */
9223#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9224 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9225 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9226#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9227 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9228 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9229 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9230 } else do { } while (0)
9231#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9232 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9233 { \
9234 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9235 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9236 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9237 } else do { } while (0)
9238#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9239 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9240 { \
9241 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9242 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9243 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9244 } else do { } while (0)
9245#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9246 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9247 { \
9248 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9249 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9250 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9251 } else do { } while (0)
9252
9253 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9254 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9255 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9256 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9257 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9258
9259 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9260 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9261 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9262 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9263 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9264 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9265 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9266 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9267 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9268 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9269 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9270 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9271 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9272 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9273 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9274 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9275 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9276 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9277 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9278 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9279 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9280 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9281 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9282 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9283 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9284 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9285 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9286 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9287 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9288 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9289 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9290 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9291 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9292 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9293 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9294 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9295
9296 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9297 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9298 {
9299 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9300 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9301 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9302 AssertRC(rc2);
9303
9304#if 0 /** @todo fix me */
9305 pDbgState->fClearCr0Mask = true;
9306 pDbgState->fClearCr4Mask = true;
9307#endif
9308 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9309 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9310 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9311 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9312 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9313 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9314 require clearing here and in the loop if we start using it. */
9315 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9316 }
9317 else
9318 {
9319 if (pDbgState->fClearCr0Mask)
9320 {
9321 pDbgState->fClearCr0Mask = false;
9322 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9323 }
9324 if (pDbgState->fClearCr4Mask)
9325 {
9326 pDbgState->fClearCr4Mask = false;
9327 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9328 }
9329 }
9330 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9331 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9332
9333 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9334 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9335 {
9336 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9337 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9338 }
9339 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9340 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9341
9342 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9343 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9344 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9345 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9346 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9348 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9349 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9350#if 0 /** @todo too slow, fix handler. */
9351 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9352#endif
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9354
9355 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9356 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9357 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9358 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9359 {
9360 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9361 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9362 }
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9364 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9365 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9366 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9367
9368 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9369 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9370 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9371 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9372 {
9373 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9374 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9375 }
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9378 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9380
9381 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9382 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9383 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9384 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9385 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9386 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9387 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9388 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9389 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9390 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9391 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9392 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9393 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9395 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9397 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9398 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9399 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9400 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9401 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9403
9404#undef IS_EITHER_ENABLED
9405#undef SET_ONLY_XBM_IF_EITHER_EN
9406#undef SET_CPE1_XBM_IF_EITHER_EN
9407#undef SET_CPEU_XBM_IF_EITHER_EN
9408#undef SET_CPE2_XBM_IF_EITHER_EN
9409
9410 /*
9411 * Sanitize the control stuff.
9412 */
9413 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9414 if (pDbgState->fCpe2Extra)
9415 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9416 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9417 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9418 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9419 {
9420 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9421 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9422 }
9423
9424 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9425 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9426 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9427 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9428}
9429
9430
9431/**
9432 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9433 *
9434 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9435 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9436 * either.
9437 *
9438 * @returns Strict VBox status code (i.e. informational status codes too).
9439 * @param pVM The cross context VM structure.
9440 * @param pVCpu The cross context virtual CPU structure.
9441 * @param pMixedCtx Pointer to the guest-CPU context.
9442 * @param pVmxTransient Pointer to the VMX-transient structure.
9443 * @param uExitReason The VM-exit reason.
9444 *
9445 * @remarks The name of this function is displayed by dtrace, so keep it short
9446 * and to the point. No longer than 33 chars long, please.
9447 */
9448static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9449 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9450{
9451 /*
9452 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9453 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9454 *
9455 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9456 * does. Must add/change/remove both places. Same ordering, please.
9457 *
9458 * Added/removed events must also be reflected in the next section
9459 * where we dispatch dtrace events.
9460 */
9461 bool fDtrace1 = false;
9462 bool fDtrace2 = false;
9463 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9464 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9465 uint32_t uEventArg = 0;
9466#define SET_EXIT(a_EventSubName) \
9467 do { \
9468 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9469 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9470 } while (0)
9471#define SET_BOTH(a_EventSubName) \
9472 do { \
9473 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9474 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9475 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9476 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9477 } while (0)
9478 switch (uExitReason)
9479 {
9480 case VMX_EXIT_MTF:
9481 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9482
9483 case VMX_EXIT_XCPT_OR_NMI:
9484 {
9485 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9486 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9487 {
9488 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9489 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9490 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9491 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9492 {
9493 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9494 {
9495 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9496 uEventArg = pVmxTransient->uExitIntErrorCode;
9497 }
9498 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9499 switch (enmEvent1)
9500 {
9501 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9502 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9503 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9504 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9505 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9506 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9507 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9508 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9509 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9510 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9511 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9512 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9513 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9514 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9515 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9516 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9517 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9518 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9519 default: break;
9520 }
9521 }
9522 else
9523 AssertFailed();
9524 break;
9525
9526 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9527 uEventArg = idxVector;
9528 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9529 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9530 break;
9531 }
9532 break;
9533 }
9534
9535 case VMX_EXIT_TRIPLE_FAULT:
9536 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9537 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9538 break;
9539 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9540 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9541 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9542 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9543 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9544
9545 /* Instruction specific VM-exits: */
9546 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9547 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9548 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9549 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9550 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9551 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9552 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9553 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9554 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9555 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9556 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9557 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9558 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9559 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9560 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9561 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9562 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9563 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9564 case VMX_EXIT_MOV_CRX:
9565 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9566/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9567* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9568 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9569 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9570 SET_BOTH(CRX_READ);
9571 else
9572 SET_BOTH(CRX_WRITE);
9573 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9574 break;
9575 case VMX_EXIT_MOV_DRX:
9576 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9577 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9578 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9579 SET_BOTH(DRX_READ);
9580 else
9581 SET_BOTH(DRX_WRITE);
9582 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9583 break;
9584 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9585 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9586 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9587 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9588 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9589 case VMX_EXIT_XDTR_ACCESS:
9590 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9591 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9592 {
9593 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9594 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9595 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9596 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9597 }
9598 break;
9599
9600 case VMX_EXIT_TR_ACCESS:
9601 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9602 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9603 {
9604 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9605 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9606 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9607 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9608 }
9609 break;
9610
9611 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9612 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9613 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9614 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9615 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9616 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9617 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9618 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9619 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9620 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9621 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9622
9623 /* Events that aren't relevant at this point. */
9624 case VMX_EXIT_EXT_INT:
9625 case VMX_EXIT_INT_WINDOW:
9626 case VMX_EXIT_NMI_WINDOW:
9627 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9628 case VMX_EXIT_PREEMPT_TIMER:
9629 case VMX_EXIT_IO_INSTR:
9630 break;
9631
9632 /* Errors and unexpected events. */
9633 case VMX_EXIT_INIT_SIGNAL:
9634 case VMX_EXIT_SIPI:
9635 case VMX_EXIT_IO_SMI:
9636 case VMX_EXIT_SMI:
9637 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9638 case VMX_EXIT_ERR_MSR_LOAD:
9639 case VMX_EXIT_ERR_MACHINE_CHECK:
9640 break;
9641
9642 default:
9643 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9644 break;
9645 }
9646#undef SET_BOTH
9647#undef SET_EXIT
9648
9649 /*
9650 * Dtrace tracepoints go first. We do them here at once so we don't
9651 * have to copy the guest state saving and stuff a few dozen times.
9652 * Down side is that we've got to repeat the switch, though this time
9653 * we use enmEvent since the probes are a subset of what DBGF does.
9654 */
9655 if (fDtrace1 || fDtrace2)
9656 {
9657 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9658 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9659 switch (enmEvent1)
9660 {
9661 /** @todo consider which extra parameters would be helpful for each probe. */
9662 case DBGFEVENT_END: break;
9663 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9665 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9666 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9667 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9668 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9669 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9671 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9672 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9673 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9674 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9675 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9676 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9677 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9678 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9679 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9680 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9681 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9682 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9683 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9684 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9685 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9686 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9688 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9689 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9690 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9691 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9692 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9693 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9694 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9695 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9696 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9697 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9698 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9699 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9700 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9701 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9703 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9704 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9705 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9706 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9707 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9708 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9709 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9710 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9711 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9712 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9713 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9714 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9715 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9716 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9717 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9718 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9719 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9720 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9721 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9722 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9723 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9724 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9725 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9726 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9727 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9728 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9729 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9730 }
9731 switch (enmEvent2)
9732 {
9733 /** @todo consider which extra parameters would be helpful for each probe. */
9734 case DBGFEVENT_END: break;
9735 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9736 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9737 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9738 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9739 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9740 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9741 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9742 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9743 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9744 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9745 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9746 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9747 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9748 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9749 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9750 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9751 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9752 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9754 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9760 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9761 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9762 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9763 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9764 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9765 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9766 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9767 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9768 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9769 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9770 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9771 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9772 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9780 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9781 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9782 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9783 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9784 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9787 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9788 }
9789 }
9790
9791 /*
9792 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9793 * the DBGF call will do a full check).
9794 *
9795 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9796 * Note! If we have to events, we prioritize the first, i.e. the instruction
9797 * one, in order to avoid event nesting.
9798 */
9799 if ( enmEvent1 != DBGFEVENT_END
9800 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9801 {
9802 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9803 if (rcStrict != VINF_SUCCESS)
9804 return rcStrict;
9805 }
9806 else if ( enmEvent2 != DBGFEVENT_END
9807 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9808 {
9809 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9810 if (rcStrict != VINF_SUCCESS)
9811 return rcStrict;
9812 }
9813
9814 return VINF_SUCCESS;
9815}
9816
9817
9818/**
9819 * Single-stepping VM-exit filtering.
9820 *
9821 * This is preprocessing the exits and deciding whether we've gotten far enough
9822 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9823 * performed.
9824 *
9825 * @returns Strict VBox status code (i.e. informational status codes too).
9826 * @param pVM The cross context VM structure.
9827 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9828 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9829 * out-of-sync. Make sure to update the required
9830 * fields before using them.
9831 * @param pVmxTransient Pointer to the VMX-transient structure.
9832 * @param uExitReason The VM-exit reason.
9833 * @param pDbgState The debug state.
9834 */
9835DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9836 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9837{
9838 /*
9839 * Expensive (saves context) generic dtrace exit probe.
9840 */
9841 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9842 { /* more likely */ }
9843 else
9844 {
9845 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9846 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9847 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9848 }
9849
9850 /*
9851 * Check for host NMI, just to get that out of the way.
9852 */
9853 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9854 { /* normally likely */ }
9855 else
9856 {
9857 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9858 AssertRCReturn(rc2, rc2);
9859 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9860 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9861 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9862 }
9863
9864 /*
9865 * Check for single stepping event if we're stepping.
9866 */
9867 if (pVCpu->hm.s.fSingleInstruction)
9868 {
9869 switch (uExitReason)
9870 {
9871 case VMX_EXIT_MTF:
9872 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9873
9874 /* Various events: */
9875 case VMX_EXIT_XCPT_OR_NMI:
9876 case VMX_EXIT_EXT_INT:
9877 case VMX_EXIT_TRIPLE_FAULT:
9878 case VMX_EXIT_INT_WINDOW:
9879 case VMX_EXIT_NMI_WINDOW:
9880 case VMX_EXIT_TASK_SWITCH:
9881 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9882 case VMX_EXIT_APIC_ACCESS:
9883 case VMX_EXIT_EPT_VIOLATION:
9884 case VMX_EXIT_EPT_MISCONFIG:
9885 case VMX_EXIT_PREEMPT_TIMER:
9886
9887 /* Instruction specific VM-exits: */
9888 case VMX_EXIT_CPUID:
9889 case VMX_EXIT_GETSEC:
9890 case VMX_EXIT_HLT:
9891 case VMX_EXIT_INVD:
9892 case VMX_EXIT_INVLPG:
9893 case VMX_EXIT_RDPMC:
9894 case VMX_EXIT_RDTSC:
9895 case VMX_EXIT_RSM:
9896 case VMX_EXIT_VMCALL:
9897 case VMX_EXIT_VMCLEAR:
9898 case VMX_EXIT_VMLAUNCH:
9899 case VMX_EXIT_VMPTRLD:
9900 case VMX_EXIT_VMPTRST:
9901 case VMX_EXIT_VMREAD:
9902 case VMX_EXIT_VMRESUME:
9903 case VMX_EXIT_VMWRITE:
9904 case VMX_EXIT_VMXOFF:
9905 case VMX_EXIT_VMXON:
9906 case VMX_EXIT_MOV_CRX:
9907 case VMX_EXIT_MOV_DRX:
9908 case VMX_EXIT_IO_INSTR:
9909 case VMX_EXIT_RDMSR:
9910 case VMX_EXIT_WRMSR:
9911 case VMX_EXIT_MWAIT:
9912 case VMX_EXIT_MONITOR:
9913 case VMX_EXIT_PAUSE:
9914 case VMX_EXIT_XDTR_ACCESS:
9915 case VMX_EXIT_TR_ACCESS:
9916 case VMX_EXIT_INVEPT:
9917 case VMX_EXIT_RDTSCP:
9918 case VMX_EXIT_INVVPID:
9919 case VMX_EXIT_WBINVD:
9920 case VMX_EXIT_XSETBV:
9921 case VMX_EXIT_RDRAND:
9922 case VMX_EXIT_INVPCID:
9923 case VMX_EXIT_VMFUNC:
9924 case VMX_EXIT_RDSEED:
9925 case VMX_EXIT_XSAVES:
9926 case VMX_EXIT_XRSTORS:
9927 {
9928 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9929 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9930 AssertRCReturn(rc2, rc2);
9931 if ( pMixedCtx->rip != pDbgState->uRipStart
9932 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9933 return VINF_EM_DBG_STEPPED;
9934 break;
9935 }
9936
9937 /* Errors and unexpected events: */
9938 case VMX_EXIT_INIT_SIGNAL:
9939 case VMX_EXIT_SIPI:
9940 case VMX_EXIT_IO_SMI:
9941 case VMX_EXIT_SMI:
9942 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9943 case VMX_EXIT_ERR_MSR_LOAD:
9944 case VMX_EXIT_ERR_MACHINE_CHECK:
9945 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9946 break;
9947
9948 default:
9949 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9950 break;
9951 }
9952 }
9953
9954 /*
9955 * Check for debugger event breakpoints and dtrace probes.
9956 */
9957 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9958 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9959 {
9960 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9961 if (rcStrict != VINF_SUCCESS)
9962 return rcStrict;
9963 }
9964
9965 /*
9966 * Normal processing.
9967 */
9968#ifdef HMVMX_USE_FUNCTION_TABLE
9969 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9970#else
9971 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9972#endif
9973}
9974
9975
9976/**
9977 * Single steps guest code using VT-x.
9978 *
9979 * @returns Strict VBox status code (i.e. informational status codes too).
9980 * @param pVM The cross context VM structure.
9981 * @param pVCpu The cross context virtual CPU structure.
9982 * @param pCtx Pointer to the guest-CPU context.
9983 *
9984 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9985 */
9986static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9987{
9988 VMXTRANSIENT VmxTransient;
9989 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9990
9991 /* Set HMCPU indicators. */
9992 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9993 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9994 pVCpu->hm.s.fDebugWantRdTscExit = false;
9995 pVCpu->hm.s.fUsingDebugLoop = true;
9996
9997 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9998 VMXRUNDBGSTATE DbgState;
9999 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10000 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10001
10002 /*
10003 * The loop.
10004 */
10005 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10006 for (uint32_t cLoops = 0; ; cLoops++)
10007 {
10008 Assert(!HMR0SuspendPending());
10009 HMVMX_ASSERT_CPU_SAFE();
10010 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10011
10012 /*
10013 * Preparatory work for running guest code, this may force us to return
10014 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10015 */
10016 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10017 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10018 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10019 if (rcStrict != VINF_SUCCESS)
10020 break;
10021
10022 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10023 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10024
10025 /*
10026 * Now we can run the guest code.
10027 */
10028 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10029
10030 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10031
10032 /*
10033 * Restore any residual host-state and save any bits shared between host
10034 * and guest into the guest-CPU state. Re-enables interrupts!
10035 */
10036 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10037
10038 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10039 if (RT_SUCCESS(rcRun))
10040 { /* very likely */ }
10041 else
10042 {
10043 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10044 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10045 return rcRun;
10046 }
10047
10048 /* Profile the VM-exit. */
10049 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10050 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10051 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10052 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10053 HMVMX_START_EXIT_DISPATCH_PROF();
10054
10055 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10056
10057 /*
10058 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10059 */
10060 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10061 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10062 if (rcStrict != VINF_SUCCESS)
10063 break;
10064 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10065 {
10066 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10067 rcStrict = VINF_EM_RAW_INTERRUPT;
10068 break;
10069 }
10070
10071 /*
10072 * Stepping: Did the RIP change, if so, consider it a single step.
10073 * Otherwise, make sure one of the TFs gets set.
10074 */
10075 if (fStepping)
10076 {
10077 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10078 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10079 AssertRCReturn(rc2, rc2);
10080 if ( pCtx->rip != DbgState.uRipStart
10081 || pCtx->cs.Sel != DbgState.uCsStart)
10082 {
10083 rcStrict = VINF_EM_DBG_STEPPED;
10084 break;
10085 }
10086 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10087 }
10088
10089 /*
10090 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10091 */
10092 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10093 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10094 }
10095
10096 /*
10097 * Clear the X86_EFL_TF if necessary.
10098 */
10099 if (pVCpu->hm.s.fClearTrapFlag)
10100 {
10101 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10102 AssertRCReturn(rc2, rc2);
10103 pVCpu->hm.s.fClearTrapFlag = false;
10104 pCtx->eflags.Bits.u1TF = 0;
10105 }
10106 /** @todo there seems to be issues with the resume flag when the monitor trap
10107 * flag is pending without being used. Seen early in bios init when
10108 * accessing APIC page in protected mode. */
10109
10110 /*
10111 * Restore VM-exit control settings as we may not reenter this function the
10112 * next time around.
10113 */
10114 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10115
10116 /* Restore HMCPU indicators. */
10117 pVCpu->hm.s.fUsingDebugLoop = false;
10118 pVCpu->hm.s.fDebugWantRdTscExit = false;
10119 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10120
10121 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10122 return rcStrict;
10123}
10124
10125
10126/** @} */
10127
10128
10129/**
10130 * Checks if any expensive dtrace probes are enabled and we should go to the
10131 * debug loop.
10132 *
10133 * @returns true if we should use debug loop, false if not.
10134 */
10135static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10136{
10137 /* It's probably faster to OR the raw 32-bit counter variables together.
10138 Since the variables are in an array and the probes are next to one
10139 another (more or less), we have good locality. So, better read
10140 eight-nine cache lines ever time and only have one conditional, than
10141 128+ conditionals, right? */
10142 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10143 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10144 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10145 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10146 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10147 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10148 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10149 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10150 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10151 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10152 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10153 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10154 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10155 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10156 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10157 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10158 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10159 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10160 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10161 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10162 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10163 ) != 0
10164 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10165 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10166 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10167 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10168 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10169 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10170 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10171 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10172 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10173 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10174 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10175 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10176 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10177 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10178 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10179 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10180 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10181 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10182 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10184 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10185 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10186 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10187 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10188 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10189 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10190 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10192 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10193 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10194 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10195 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10196 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10197 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10198 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10199 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10200 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10201 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10202 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10203 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10204 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10205 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10206 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10207 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10208 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10209 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10210 ) != 0
10211 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10212 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10213 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10214 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10215 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10216 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10217 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10218 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10219 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10220 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10221 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10222 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10223 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10224 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10225 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10226 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10227 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10228 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10229 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10230 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10232 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10233 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10234 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10235 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10236 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10237 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10238 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10240 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10241 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10242 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10243 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10244 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10245 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10246 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10247 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10248 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10249 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10250 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10251 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10252 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10253 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10254 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10255 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10256 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10257 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10258 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10259 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10260 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10261 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10262 ) != 0;
10263}
10264
10265
10266/**
10267 * Runs the guest code using VT-x.
10268 *
10269 * @returns Strict VBox status code (i.e. informational status codes too).
10270 * @param pVM The cross context VM structure.
10271 * @param pVCpu The cross context virtual CPU structure.
10272 * @param pCtx Pointer to the guest-CPU context.
10273 */
10274VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10275{
10276 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10277 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10278 HMVMX_ASSERT_PREEMPT_SAFE();
10279
10280 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10281
10282 VBOXSTRICTRC rcStrict;
10283 if ( !pVCpu->hm.s.fUseDebugLoop
10284 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10285 && !DBGFIsStepping(pVCpu) )
10286 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10287 else
10288 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10289
10290 if (rcStrict == VERR_EM_INTERPRETER)
10291 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10292 else if (rcStrict == VINF_EM_RESET)
10293 rcStrict = VINF_EM_TRIPLE_FAULT;
10294
10295 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10296 if (RT_FAILURE(rc2))
10297 {
10298 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10299 rcStrict = rc2;
10300 }
10301 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10302 return rcStrict;
10303}
10304
10305
10306#ifndef HMVMX_USE_FUNCTION_TABLE
10307DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10308{
10309# ifdef DEBUG_ramshankar
10310# define RETURN_EXIT_CALL(a_CallExpr) \
10311 do { \
10312 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10313 VBOXSTRICTRC rcStrict = a_CallExpr; \
10314 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10315 return rcStrict; \
10316 } while (0)
10317# else
10318# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10319# endif
10320 switch (rcReason)
10321 {
10322 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10323 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10324 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10325 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10326 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10327 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10328 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10329 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10330 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10331 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10332 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10333 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10334 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10335 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10336 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10337 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10338 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10339 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10340 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10341 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10342 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10343 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10344 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10345 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10346 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10347 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10348 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10349 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10350 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10351 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10352 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10353 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10354 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10355 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10356
10357 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10358 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10359 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10360 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10361 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10362 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10363 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10364 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10365 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10366
10367 case VMX_EXIT_VMCLEAR:
10368 case VMX_EXIT_VMLAUNCH:
10369 case VMX_EXIT_VMPTRLD:
10370 case VMX_EXIT_VMPTRST:
10371 case VMX_EXIT_VMREAD:
10372 case VMX_EXIT_VMRESUME:
10373 case VMX_EXIT_VMWRITE:
10374 case VMX_EXIT_VMXOFF:
10375 case VMX_EXIT_VMXON:
10376 case VMX_EXIT_INVEPT:
10377 case VMX_EXIT_INVVPID:
10378 case VMX_EXIT_VMFUNC:
10379 case VMX_EXIT_XSAVES:
10380 case VMX_EXIT_XRSTORS:
10381 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10382 case VMX_EXIT_RESERVED_60:
10383 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10384 case VMX_EXIT_RESERVED_62:
10385 default:
10386 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10387 }
10388#undef RETURN_EXIT_CALL
10389}
10390#endif /* !HMVMX_USE_FUNCTION_TABLE */
10391
10392
10393#ifdef VBOX_STRICT
10394/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10395# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10396 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10397
10398# define HMVMX_ASSERT_PREEMPT_CPUID() \
10399 do { \
10400 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10401 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10402 } while (0)
10403
10404# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10405 do { \
10406 AssertPtr(pVCpu); \
10407 AssertPtr(pMixedCtx); \
10408 AssertPtr(pVmxTransient); \
10409 Assert(pVmxTransient->fVMEntryFailed == false); \
10410 Assert(ASMIntAreEnabled()); \
10411 HMVMX_ASSERT_PREEMPT_SAFE(); \
10412 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10413 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10414 HMVMX_ASSERT_PREEMPT_SAFE(); \
10415 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10416 HMVMX_ASSERT_PREEMPT_CPUID(); \
10417 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10418 } while (0)
10419
10420# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10421 do { \
10422 Log4Func(("\n")); \
10423 } while (0)
10424#else /* nonstrict builds: */
10425# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10426 do { \
10427 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10428 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10429 } while (0)
10430# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10431#endif
10432
10433
10434/**
10435 * Advances the guest RIP by the specified number of bytes.
10436 *
10437 * @param pVCpu The cross context virtual CPU structure.
10438 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10439 * out-of-sync. Make sure to update the required fields
10440 * before using them.
10441 * @param cbInstr Number of bytes to advance the RIP by.
10442 *
10443 * @remarks No-long-jump zone!!!
10444 */
10445DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10446{
10447 /* Advance the RIP. */
10448 pMixedCtx->rip += cbInstr;
10449 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10450
10451 /* Update interrupt inhibition. */
10452 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10453 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10454 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10455}
10456
10457
10458/**
10459 * Advances the guest RIP after reading it from the VMCS.
10460 *
10461 * @returns VBox status code, no informational status codes.
10462 * @param pVCpu The cross context virtual CPU structure.
10463 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10464 * out-of-sync. Make sure to update the required fields
10465 * before using them.
10466 * @param pVmxTransient Pointer to the VMX transient structure.
10467 *
10468 * @remarks No-long-jump zone!!!
10469 */
10470static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10471{
10472 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10473 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10474 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10475 AssertRCReturn(rc, rc);
10476
10477 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10478
10479 /*
10480 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10481 * pending debug exception field as it takes care of priority of events.
10482 *
10483 * See Intel spec. 32.2.1 "Debug Exceptions".
10484 */
10485 if ( !pVCpu->hm.s.fSingleInstruction
10486 && pMixedCtx->eflags.Bits.u1TF)
10487 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10488
10489 return VINF_SUCCESS;
10490}
10491
10492
10493/**
10494 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10495 * and update error record fields accordingly.
10496 *
10497 * @return VMX_IGS_* return codes.
10498 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10499 * wrong with the guest state.
10500 *
10501 * @param pVM The cross context VM structure.
10502 * @param pVCpu The cross context virtual CPU structure.
10503 * @param pCtx Pointer to the guest-CPU state.
10504 *
10505 * @remarks This function assumes our cache of the VMCS controls
10506 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10507 */
10508static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10509{
10510#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10511#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10512 uError = (err); \
10513 break; \
10514 } else do { } while (0)
10515
10516 int rc;
10517 uint32_t uError = VMX_IGS_ERROR;
10518 uint32_t u32Val;
10519 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10520
10521 do
10522 {
10523 /*
10524 * CR0.
10525 */
10526 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10527 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10528 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10529 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10530 if (fUnrestrictedGuest)
10531 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10532
10533 uint32_t u32GuestCR0;
10534 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10535 AssertRCBreak(rc);
10536 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10537 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10538 if ( !fUnrestrictedGuest
10539 && (u32GuestCR0 & X86_CR0_PG)
10540 && !(u32GuestCR0 & X86_CR0_PE))
10541 {
10542 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10543 }
10544
10545 /*
10546 * CR4.
10547 */
10548 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10549 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10550
10551 uint32_t u32GuestCR4;
10552 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10553 AssertRCBreak(rc);
10554 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10555 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10556
10557 /*
10558 * IA32_DEBUGCTL MSR.
10559 */
10560 uint64_t u64Val;
10561 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10562 AssertRCBreak(rc);
10563 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10564 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10565 {
10566 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10567 }
10568 uint64_t u64DebugCtlMsr = u64Val;
10569
10570#ifdef VBOX_STRICT
10571 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10572 AssertRCBreak(rc);
10573 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10574#endif
10575 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10576
10577 /*
10578 * RIP and RFLAGS.
10579 */
10580 uint32_t u32Eflags;
10581#if HC_ARCH_BITS == 64
10582 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10583 AssertRCBreak(rc);
10584 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10585 if ( !fLongModeGuest
10586 || !pCtx->cs.Attr.n.u1Long)
10587 {
10588 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10589 }
10590 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10591 * must be identical if the "IA-32e mode guest" VM-entry
10592 * control is 1 and CS.L is 1. No check applies if the
10593 * CPU supports 64 linear-address bits. */
10594
10595 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10596 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10597 AssertRCBreak(rc);
10598 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10599 VMX_IGS_RFLAGS_RESERVED);
10600 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10601 u32Eflags = u64Val;
10602#else
10603 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10604 AssertRCBreak(rc);
10605 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10606 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10607#endif
10608
10609 if ( fLongModeGuest
10610 || ( fUnrestrictedGuest
10611 && !(u32GuestCR0 & X86_CR0_PE)))
10612 {
10613 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10614 }
10615
10616 uint32_t u32EntryInfo;
10617 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10618 AssertRCBreak(rc);
10619 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10620 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10621 {
10622 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10623 }
10624
10625 /*
10626 * 64-bit checks.
10627 */
10628#if HC_ARCH_BITS == 64
10629 if (fLongModeGuest)
10630 {
10631 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10632 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10633 }
10634
10635 if ( !fLongModeGuest
10636 && (u32GuestCR4 & X86_CR4_PCIDE))
10637 {
10638 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10639 }
10640
10641 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10642 * 51:32 beyond the processor's physical-address width are 0. */
10643
10644 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10645 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10646 {
10647 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10648 }
10649
10650 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10651 AssertRCBreak(rc);
10652 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10653
10654 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10655 AssertRCBreak(rc);
10656 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10657#endif
10658
10659 /*
10660 * PERF_GLOBAL MSR.
10661 */
10662 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10663 {
10664 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10665 AssertRCBreak(rc);
10666 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10667 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10668 }
10669
10670 /*
10671 * PAT MSR.
10672 */
10673 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10674 {
10675 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10676 AssertRCBreak(rc);
10677 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10678 for (unsigned i = 0; i < 8; i++)
10679 {
10680 uint8_t u8Val = (u64Val & 0xff);
10681 if ( u8Val != 0 /* UC */
10682 && u8Val != 1 /* WC */
10683 && u8Val != 4 /* WT */
10684 && u8Val != 5 /* WP */
10685 && u8Val != 6 /* WB */
10686 && u8Val != 7 /* UC- */)
10687 {
10688 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10689 }
10690 u64Val >>= 8;
10691 }
10692 }
10693
10694 /*
10695 * EFER MSR.
10696 */
10697 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10698 {
10699 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10700 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10701 AssertRCBreak(rc);
10702 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10703 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10704 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10705 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10706 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10707 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10708 || !(u32GuestCR0 & X86_CR0_PG)
10709 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10710 VMX_IGS_EFER_LMA_LME_MISMATCH);
10711 }
10712
10713 /*
10714 * Segment registers.
10715 */
10716 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10717 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10718 if (!(u32Eflags & X86_EFL_VM))
10719 {
10720 /* CS */
10721 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10722 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10723 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10724 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10725 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10726 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10727 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10728 /* CS cannot be loaded with NULL in protected mode. */
10729 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10730 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10731 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10732 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10733 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10734 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10735 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10736 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10737 else
10738 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10739
10740 /* SS */
10741 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10742 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10743 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10744 if ( !(pCtx->cr0 & X86_CR0_PE)
10745 || pCtx->cs.Attr.n.u4Type == 3)
10746 {
10747 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10748 }
10749 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10750 {
10751 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10752 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10753 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10754 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10755 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10756 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10757 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10758 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10759 }
10760
10761 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10762 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10763 {
10764 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10765 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10766 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10767 || pCtx->ds.Attr.n.u4Type > 11
10768 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10769 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10770 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10771 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10772 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10773 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10774 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10775 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10776 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10777 }
10778 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10779 {
10780 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10781 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10782 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10783 || pCtx->es.Attr.n.u4Type > 11
10784 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10785 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10786 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10787 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10788 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10789 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10790 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10791 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10792 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10793 }
10794 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10795 {
10796 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10797 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10798 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10799 || pCtx->fs.Attr.n.u4Type > 11
10800 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10801 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10802 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10803 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10804 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10805 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10806 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10807 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10808 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10809 }
10810 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10811 {
10812 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10813 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10814 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10815 || pCtx->gs.Attr.n.u4Type > 11
10816 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10817 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10818 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10819 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10820 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10821 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10822 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10823 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10824 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10825 }
10826 /* 64-bit capable CPUs. */
10827#if HC_ARCH_BITS == 64
10828 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10829 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10830 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10831 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10832 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10833 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10834 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10835 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10836 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10837 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10838 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10839#endif
10840 }
10841 else
10842 {
10843 /* V86 mode checks. */
10844 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10845 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10846 {
10847 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10848 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10849 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10850 }
10851 else
10852 {
10853 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10854 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10855 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10856 }
10857
10858 /* CS */
10859 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10860 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10861 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10862 /* SS */
10863 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10864 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10865 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10866 /* DS */
10867 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10868 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10869 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10870 /* ES */
10871 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10872 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10873 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10874 /* FS */
10875 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10876 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10877 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10878 /* GS */
10879 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10880 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10881 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10882 /* 64-bit capable CPUs. */
10883#if HC_ARCH_BITS == 64
10884 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10885 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10886 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10887 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10888 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10889 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10890 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10891 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10892 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10893 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10894 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10895#endif
10896 }
10897
10898 /*
10899 * TR.
10900 */
10901 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10902 /* 64-bit capable CPUs. */
10903#if HC_ARCH_BITS == 64
10904 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10905#endif
10906 if (fLongModeGuest)
10907 {
10908 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10909 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10910 }
10911 else
10912 {
10913 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10914 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10915 VMX_IGS_TR_ATTR_TYPE_INVALID);
10916 }
10917 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10918 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10919 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10920 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10921 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10922 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10923 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10924 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10925
10926 /*
10927 * GDTR and IDTR.
10928 */
10929#if HC_ARCH_BITS == 64
10930 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10931 AssertRCBreak(rc);
10932 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10933
10934 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10935 AssertRCBreak(rc);
10936 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10937#endif
10938
10939 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10940 AssertRCBreak(rc);
10941 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10942
10943 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10944 AssertRCBreak(rc);
10945 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10946
10947 /*
10948 * Guest Non-Register State.
10949 */
10950 /* Activity State. */
10951 uint32_t u32ActivityState;
10952 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10953 AssertRCBreak(rc);
10954 HMVMX_CHECK_BREAK( !u32ActivityState
10955 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10956 VMX_IGS_ACTIVITY_STATE_INVALID);
10957 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10958 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10959 uint32_t u32IntrState;
10960 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10961 AssertRCBreak(rc);
10962 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10963 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10964 {
10965 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10966 }
10967
10968 /** @todo Activity state and injecting interrupts. Left as a todo since we
10969 * currently don't use activity states but ACTIVE. */
10970
10971 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10972 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10973
10974 /* Guest interruptibility-state. */
10975 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10976 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10977 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10978 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10979 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10980 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10981 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10982 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10983 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10984 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10985 {
10986 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10987 {
10988 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10989 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10990 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10991 }
10992 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10993 {
10994 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10995 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10996 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10997 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10998 }
10999 }
11000 /** @todo Assumes the processor is not in SMM. */
11001 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11002 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11003 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11004 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11005 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11006 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11007 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11008 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11009 {
11010 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11011 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11012 }
11013
11014 /* Pending debug exceptions. */
11015#if HC_ARCH_BITS == 64
11016 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11017 AssertRCBreak(rc);
11018 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11019 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11020 u32Val = u64Val; /* For pending debug exceptions checks below. */
11021#else
11022 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11023 AssertRCBreak(rc);
11024 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11025 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11026#endif
11027
11028 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11029 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11030 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11031 {
11032 if ( (u32Eflags & X86_EFL_TF)
11033 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11034 {
11035 /* Bit 14 is PendingDebug.BS. */
11036 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11037 }
11038 if ( !(u32Eflags & X86_EFL_TF)
11039 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11040 {
11041 /* Bit 14 is PendingDebug.BS. */
11042 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11043 }
11044 }
11045
11046 /* VMCS link pointer. */
11047 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11048 AssertRCBreak(rc);
11049 if (u64Val != UINT64_C(0xffffffffffffffff))
11050 {
11051 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11052 /** @todo Bits beyond the processor's physical-address width MBZ. */
11053 /** @todo 32-bit located in memory referenced by value of this field (as a
11054 * physical address) must contain the processor's VMCS revision ID. */
11055 /** @todo SMM checks. */
11056 }
11057
11058 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11059 * not using Nested Paging? */
11060 if ( pVM->hm.s.fNestedPaging
11061 && !fLongModeGuest
11062 && CPUMIsGuestInPAEModeEx(pCtx))
11063 {
11064 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11065 AssertRCBreak(rc);
11066 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11067
11068 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11069 AssertRCBreak(rc);
11070 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11071
11072 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11073 AssertRCBreak(rc);
11074 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11075
11076 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11077 AssertRCBreak(rc);
11078 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11079 }
11080
11081 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11082 if (uError == VMX_IGS_ERROR)
11083 uError = VMX_IGS_REASON_NOT_FOUND;
11084 } while (0);
11085
11086 pVCpu->hm.s.u32HMError = uError;
11087 return uError;
11088
11089#undef HMVMX_ERROR_BREAK
11090#undef HMVMX_CHECK_BREAK
11091}
11092
11093/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11094/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11095/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11096
11097/** @name VM-exit handlers.
11098 * @{
11099 */
11100
11101/**
11102 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11103 */
11104HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11105{
11106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11108 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11109 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11110 return VINF_SUCCESS;
11111 return VINF_EM_RAW_INTERRUPT;
11112}
11113
11114
11115/**
11116 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11117 */
11118HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11119{
11120 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11121 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11122
11123 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11124 AssertRCReturn(rc, rc);
11125
11126 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11127 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11128 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11129 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11130
11131 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11132 {
11133 /*
11134 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11135 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11136 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11137 *
11138 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11139 */
11140 VMXDispatchHostNmi();
11141 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11142 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11143 return VINF_SUCCESS;
11144 }
11145
11146 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11147 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11148 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11149 { /* likely */ }
11150 else
11151 {
11152 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11153 rcStrictRc1 = VINF_SUCCESS;
11154 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11155 return rcStrictRc1;
11156 }
11157
11158 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11159 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11160 switch (uIntType)
11161 {
11162 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11163 Assert(uVector == X86_XCPT_DB);
11164 /* no break */
11165 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11166 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11167 /* no break */
11168 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11169 {
11170 switch (uVector)
11171 {
11172 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11173 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11174 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11175 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11176 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11177 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11178 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11179
11180 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11181 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11182 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11183 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11184 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11185 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11186 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11187 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11188 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11189 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11190 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11191 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11192 default:
11193 {
11194 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11195 AssertRCReturn(rc, rc);
11196
11197 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11198 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11199 {
11200 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11201 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11202 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11203
11204 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11205 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11206 AssertRCReturn(rc, rc);
11207 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11208 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11209 0 /* GCPtrFaultAddress */);
11210 AssertRCReturn(rc, rc);
11211 }
11212 else
11213 {
11214 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11215 pVCpu->hm.s.u32HMError = uVector;
11216 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11217 }
11218 break;
11219 }
11220 }
11221 break;
11222 }
11223
11224 default:
11225 {
11226 pVCpu->hm.s.u32HMError = uExitIntInfo;
11227 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11228 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11229 break;
11230 }
11231 }
11232 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11233 return rc;
11234}
11235
11236
11237/**
11238 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11239 */
11240HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11241{
11242 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11243
11244 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11245 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11246
11247 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11249 return VINF_SUCCESS;
11250}
11251
11252
11253/**
11254 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11255 */
11256HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257{
11258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11260 {
11261 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11262 HMVMX_RETURN_UNEXPECTED_EXIT();
11263 }
11264
11265 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11266
11267 /*
11268 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11269 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11270 */
11271 uint32_t uIntrState = 0;
11272 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11273 AssertRCReturn(rc, rc);
11274
11275 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11276 if ( fBlockSti
11277 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11278 {
11279 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11280 }
11281
11282 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11283 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11284
11285 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11286 return VINF_SUCCESS;
11287}
11288
11289
11290/**
11291 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11292 */
11293HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11294{
11295 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11297 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11298}
11299
11300
11301/**
11302 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11303 */
11304HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11305{
11306 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11307 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11308 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11309}
11310
11311
11312/**
11313 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11314 */
11315HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11316{
11317 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11318 PVM pVM = pVCpu->CTX_SUFF(pVM);
11319 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11320 if (RT_LIKELY(rc == VINF_SUCCESS))
11321 {
11322 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11323 Assert(pVmxTransient->cbInstr == 2);
11324 }
11325 else
11326 {
11327 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11328 rc = VERR_EM_INTERPRETER;
11329 }
11330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11331 return rc;
11332}
11333
11334
11335/**
11336 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11337 */
11338HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11339{
11340 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11341 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11342 AssertRCReturn(rc, rc);
11343
11344 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11345 return VINF_EM_RAW_EMULATE_INSTR;
11346
11347 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11348 HMVMX_RETURN_UNEXPECTED_EXIT();
11349}
11350
11351
11352/**
11353 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11354 */
11355HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11356{
11357 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11358 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11359 AssertRCReturn(rc, rc);
11360
11361 PVM pVM = pVCpu->CTX_SUFF(pVM);
11362 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11363 if (RT_LIKELY(rc == VINF_SUCCESS))
11364 {
11365 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11366 Assert(pVmxTransient->cbInstr == 2);
11367 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11368 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11369 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11370 }
11371 else
11372 rc = VERR_EM_INTERPRETER;
11373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11374 return rc;
11375}
11376
11377
11378/**
11379 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11380 */
11381HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11382{
11383 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11384 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11385 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11386 AssertRCReturn(rc, rc);
11387
11388 PVM pVM = pVCpu->CTX_SUFF(pVM);
11389 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11390 if (RT_SUCCESS(rc))
11391 {
11392 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11393 Assert(pVmxTransient->cbInstr == 3);
11394 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11396 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11397 }
11398 else
11399 {
11400 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11401 rc = VERR_EM_INTERPRETER;
11402 }
11403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11404 return rc;
11405}
11406
11407
11408/**
11409 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11410 */
11411HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11412{
11413 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11414 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
11415 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
11416 AssertRCReturn(rc, rc);
11417
11418 PVM pVM = pVCpu->CTX_SUFF(pVM);
11419 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11420 if (RT_LIKELY(rc == VINF_SUCCESS))
11421 {
11422 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11423 Assert(pVmxTransient->cbInstr == 2);
11424 }
11425 else
11426 {
11427 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11428 rc = VERR_EM_INTERPRETER;
11429 }
11430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11431 return rc;
11432}
11433
11434
11435/**
11436 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11437 */
11438HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11439{
11440 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11441 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11442
11443 if (pVCpu->hm.s.fHypercallsEnabled)
11444 {
11445#if 0
11446 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11447#else
11448 /* Aggressive state sync. for now. */
11449 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11450 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11451#endif
11452 rc |= hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11453 AssertRCReturn(rc, rc);
11454
11455 /** @todo pre-increment RIP before hypercall will break when we have to implement
11456 * continuing hypercalls (e.g. Hyper-V). */
11457 /** @todo r=bird: GIMHypercall will probably have to be able to return
11458 * informational status codes, so it should be made VBOXSTRICTRC. Not
11459 * doing that now because the status code handling isn't clean (i.e.
11460 * if you use RT_SUCCESS(rc) on the result of something, you don't
11461 * return rc in the success case, you return VINF_SUCCESS). */
11462 rc = GIMHypercall(pVCpu, pMixedCtx);
11463 /* If the hypercall changes anything other than guest general-purpose registers,
11464 we would need to reload the guest changed bits here before VM-entry. */
11465 return rc;
11466 }
11467
11468 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11469 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11470 return VINF_SUCCESS;
11471}
11472
11473
11474/**
11475 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11476 */
11477HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11478{
11479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11480 PVM pVM = pVCpu->CTX_SUFF(pVM);
11481 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11482
11483 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11484 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11485 AssertRCReturn(rc, rc);
11486
11487 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11488 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11489 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11490 else
11491 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11492 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11493 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11494 return rcStrict;
11495}
11496
11497
11498/**
11499 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11500 */
11501HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11502{
11503 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11504 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11505 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11506 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11507 AssertRCReturn(rc, rc);
11508
11509 PVM pVM = pVCpu->CTX_SUFF(pVM);
11510 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11511 if (RT_LIKELY(rc == VINF_SUCCESS))
11512 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11513 else
11514 {
11515 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11516 rc = VERR_EM_INTERPRETER;
11517 }
11518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11519 return rc;
11520}
11521
11522
11523/**
11524 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11525 */
11526HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11527{
11528 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11529 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11530 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11531 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11532 AssertRCReturn(rc, rc);
11533
11534 PVM pVM = pVCpu->CTX_SUFF(pVM);
11535 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11536 rc = VBOXSTRICTRC_VAL(rc2);
11537 if (RT_LIKELY( rc == VINF_SUCCESS
11538 || rc == VINF_EM_HALT))
11539 {
11540 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11541 AssertRCReturn(rc3, rc3);
11542
11543 if ( rc == VINF_EM_HALT
11544 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11545 {
11546 rc = VINF_SUCCESS;
11547 }
11548 }
11549 else
11550 {
11551 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11552 rc = VERR_EM_INTERPRETER;
11553 }
11554 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11555 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11557 return rc;
11558}
11559
11560
11561/**
11562 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11563 */
11564HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11565{
11566 /*
11567 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11568 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11569 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11570 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11571 */
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11573 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11574 HMVMX_RETURN_UNEXPECTED_EXIT();
11575}
11576
11577
11578/**
11579 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11580 */
11581HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11582{
11583 /*
11584 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11585 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11586 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11587 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11588 */
11589 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11590 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11591 HMVMX_RETURN_UNEXPECTED_EXIT();
11592}
11593
11594
11595/**
11596 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11597 */
11598HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11599{
11600 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11601 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11602 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11603 HMVMX_RETURN_UNEXPECTED_EXIT();
11604}
11605
11606
11607/**
11608 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11609 */
11610HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11611{
11612 /*
11613 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11614 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11615 * See Intel spec. 25.3 "Other Causes of VM-exits".
11616 */
11617 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11618 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11619 HMVMX_RETURN_UNEXPECTED_EXIT();
11620}
11621
11622
11623/**
11624 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11625 * VM-exit.
11626 */
11627HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11628{
11629 /*
11630 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11631 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11632 *
11633 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11634 * See Intel spec. "23.8 Restrictions on VMX operation".
11635 */
11636 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11637 return VINF_SUCCESS;
11638}
11639
11640
11641/**
11642 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11643 * VM-exit.
11644 */
11645HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11646{
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11648 return VINF_EM_RESET;
11649}
11650
11651
11652/**
11653 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11654 */
11655HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11656{
11657 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11658 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11659
11660 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11661 AssertRCReturn(rc, rc);
11662
11663 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11664 rc = VINF_SUCCESS;
11665 else
11666 rc = VINF_EM_HALT;
11667
11668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11669 if (rc != VINF_SUCCESS)
11670 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11671 return rc;
11672}
11673
11674
11675/**
11676 * VM-exit handler for instructions that result in a \#UD exception delivered to
11677 * the guest.
11678 */
11679HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11680{
11681 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11682 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11683 return VINF_SUCCESS;
11684}
11685
11686
11687/**
11688 * VM-exit handler for expiry of the VMX preemption timer.
11689 */
11690HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11691{
11692 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11693
11694 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11695 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11696
11697 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11698 PVM pVM = pVCpu->CTX_SUFF(pVM);
11699 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11701 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11702}
11703
11704
11705/**
11706 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11707 */
11708HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11709{
11710 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11711
11712 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11713 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11714 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11715 AssertRCReturn(rc, rc);
11716
11717 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11718 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11719
11720 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11721
11722 return rcStrict;
11723}
11724
11725
11726/**
11727 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11728 */
11729HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11730{
11731 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11732
11733 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11734 /** @todo implement EMInterpretInvpcid() */
11735 return VERR_EM_INTERPRETER;
11736}
11737
11738
11739/**
11740 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11741 * Error VM-exit.
11742 */
11743HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11746 AssertRCReturn(rc, rc);
11747
11748 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11749 AssertRCReturn(rc, rc);
11750
11751 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11752 NOREF(uInvalidReason);
11753
11754#ifdef VBOX_STRICT
11755 uint32_t uIntrState;
11756 RTHCUINTREG uHCReg;
11757 uint64_t u64Val;
11758 uint32_t u32Val;
11759
11760 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11761 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11762 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11763 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11764 AssertRCReturn(rc, rc);
11765
11766 Log4(("uInvalidReason %u\n", uInvalidReason));
11767 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11768 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11769 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11770 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11771
11772 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11773 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11774 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11775 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11776 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11777 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11778 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11779 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11780 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11781 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11782 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11783 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11784#else
11785 NOREF(pVmxTransient);
11786#endif
11787
11788 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11789 return VERR_VMX_INVALID_GUEST_STATE;
11790}
11791
11792
11793/**
11794 * VM-exit handler for VM-entry failure due to an MSR-load
11795 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11796 */
11797HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11798{
11799 NOREF(pVmxTransient);
11800 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11801 HMVMX_RETURN_UNEXPECTED_EXIT();
11802}
11803
11804
11805/**
11806 * VM-exit handler for VM-entry failure due to a machine-check event
11807 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11808 */
11809HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11810{
11811 NOREF(pVmxTransient);
11812 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11813 HMVMX_RETURN_UNEXPECTED_EXIT();
11814}
11815
11816
11817/**
11818 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11819 * theory.
11820 */
11821HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11822{
11823 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11824 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11825 return VERR_VMX_UNDEFINED_EXIT_CODE;
11826}
11827
11828
11829/**
11830 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11831 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11832 * Conditional VM-exit.
11833 */
11834HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11835{
11836 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11837
11838 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11839 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11840 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11841 return VERR_EM_INTERPRETER;
11842 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11843 HMVMX_RETURN_UNEXPECTED_EXIT();
11844}
11845
11846
11847/**
11848 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11849 */
11850HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11851{
11852 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11853
11854 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11856 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11857 return VERR_EM_INTERPRETER;
11858 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11859 HMVMX_RETURN_UNEXPECTED_EXIT();
11860}
11861
11862
11863/**
11864 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11865 */
11866HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11867{
11868 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11869
11870 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11871 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11872 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11873 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11874 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11875 {
11876 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11877 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11878 }
11879 AssertRCReturn(rc, rc);
11880 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11881
11882#ifdef VBOX_STRICT
11883 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11884 {
11885 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11886 && pMixedCtx->ecx != MSR_K6_EFER)
11887 {
11888 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11889 pMixedCtx->ecx));
11890 HMVMX_RETURN_UNEXPECTED_EXIT();
11891 }
11892 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11893 {
11894 VMXMSREXITREAD enmRead;
11895 VMXMSREXITWRITE enmWrite;
11896 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11897 AssertRCReturn(rc2, rc2);
11898 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11899 {
11900 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11901 HMVMX_RETURN_UNEXPECTED_EXIT();
11902 }
11903 }
11904 }
11905#endif
11906
11907 PVM pVM = pVCpu->CTX_SUFF(pVM);
11908 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11909 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11910 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11912 if (RT_SUCCESS(rc))
11913 {
11914 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11915 Assert(pVmxTransient->cbInstr == 2);
11916 }
11917 return rc;
11918}
11919
11920
11921/**
11922 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11923 */
11924HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11925{
11926 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11927 PVM pVM = pVCpu->CTX_SUFF(pVM);
11928 int rc = VINF_SUCCESS;
11929
11930 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11931 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11932 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11933 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11934 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11935 {
11936 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11937 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11938 }
11939 AssertRCReturn(rc, rc);
11940 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11941
11942 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11943 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11945
11946 if (RT_SUCCESS(rc))
11947 {
11948 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11949
11950 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11951 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11952 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11953 {
11954 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11955 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11956 EMInterpretWrmsr() changes it. */
11957 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
11958 }
11959 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11960 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11961 else if (pMixedCtx->ecx == MSR_K6_EFER)
11962 {
11963 /*
11964 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11965 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11966 * the other bits as well, SCE and NXE. See @bugref{7368}.
11967 */
11968 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
11969 }
11970
11971 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11972 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11973 {
11974 switch (pMixedCtx->ecx)
11975 {
11976 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11977 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11978 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11979 case MSR_K8_FS_BASE: /* no break */
11980 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
11981 case MSR_K6_EFER: /* already handled above */ break;
11982 default:
11983 {
11984 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11985 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11986 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11987 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
11988 break;
11989 }
11990 }
11991 }
11992#ifdef VBOX_STRICT
11993 else
11994 {
11995 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11996 switch (pMixedCtx->ecx)
11997 {
11998 case MSR_IA32_SYSENTER_CS:
11999 case MSR_IA32_SYSENTER_EIP:
12000 case MSR_IA32_SYSENTER_ESP:
12001 case MSR_K8_FS_BASE:
12002 case MSR_K8_GS_BASE:
12003 {
12004 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12005 HMVMX_RETURN_UNEXPECTED_EXIT();
12006 }
12007
12008 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12009 default:
12010 {
12011 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12012 {
12013 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12014 if (pMixedCtx->ecx != MSR_K6_EFER)
12015 {
12016 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12017 pMixedCtx->ecx));
12018 HMVMX_RETURN_UNEXPECTED_EXIT();
12019 }
12020 }
12021
12022 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12023 {
12024 VMXMSREXITREAD enmRead;
12025 VMXMSREXITWRITE enmWrite;
12026 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12027 AssertRCReturn(rc2, rc2);
12028 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12029 {
12030 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12031 HMVMX_RETURN_UNEXPECTED_EXIT();
12032 }
12033 }
12034 break;
12035 }
12036 }
12037 }
12038#endif /* VBOX_STRICT */
12039 }
12040 return rc;
12041}
12042
12043
12044/**
12045 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12046 */
12047HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12048{
12049 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12050
12051 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12052 return VINF_EM_RAW_INTERRUPT;
12053}
12054
12055
12056/**
12057 * VM-exit handler for when the TPR value is lowered below the specified
12058 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12059 */
12060HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12061{
12062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12063 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12064
12065 /*
12066 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12067 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12068 * resume guest execution.
12069 */
12070 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12072 return VINF_SUCCESS;
12073}
12074
12075
12076/**
12077 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12078 * VM-exit.
12079 *
12080 * @retval VINF_SUCCESS when guest execution can continue.
12081 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12082 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12083 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12084 * interpreter.
12085 */
12086HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12087{
12088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12089 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12090 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12091 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12092 AssertRCReturn(rc, rc);
12093
12094 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12095 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12096 PVM pVM = pVCpu->CTX_SUFF(pVM);
12097 VBOXSTRICTRC rcStrict;
12098 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12099 switch (uAccessType)
12100 {
12101 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12102 {
12103 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12104 AssertRCReturn(rc, rc);
12105
12106 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12107 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12108 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12109 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12110 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12111 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12112 {
12113 case 0: /* CR0 */
12114 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12115 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12116 break;
12117 case 2: /* CR2 */
12118 /* Nothing to do here, CR2 it's not part of the VMCS. */
12119 break;
12120 case 3: /* CR3 */
12121 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12123 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12124 break;
12125 case 4: /* CR4 */
12126 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12127 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12128 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12129 break;
12130 case 8: /* CR8 */
12131 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12132 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12133 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12134 break;
12135 default:
12136 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12137 break;
12138 }
12139
12140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12141 break;
12142 }
12143
12144 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12145 {
12146 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12147 AssertRCReturn(rc, rc);
12148
12149 Assert( !pVM->hm.s.fNestedPaging
12150 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12151 || pVCpu->hm.s.fUsingDebugLoop
12152 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12153
12154 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12155 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12156 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12157
12158 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12159 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12160 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12161 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12163 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12164 VBOXSTRICTRC_VAL(rcStrict)));
12165 break;
12166 }
12167
12168 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12169 {
12170 AssertRCReturn(rc, rc);
12171 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12172 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12173 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12174 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12175 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12176 break;
12177 }
12178
12179 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12180 {
12181 AssertRCReturn(rc, rc);
12182 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12183 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12184 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12185 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12187 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12188 break;
12189 }
12190
12191 default:
12192 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12193 VERR_VMX_UNEXPECTED_EXCEPTION);
12194 }
12195
12196 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12197 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12198 NOREF(pVM);
12199 return rcStrict;
12200}
12201
12202
12203/**
12204 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12205 * VM-exit.
12206 */
12207HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12208{
12209 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12210 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12211
12212 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12213 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12214 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12215 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12216 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12217 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12218 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12219 AssertRCReturn(rc2, rc2);
12220
12221 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12222 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12223 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12224 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12225 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12226 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12227 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12228 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12229 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12230
12231 /* I/O operation lookup arrays. */
12232 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12233 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12234
12235 VBOXSTRICTRC rcStrict;
12236 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12237 uint32_t const cbInstr = pVmxTransient->cbInstr;
12238 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12239 PVM pVM = pVCpu->CTX_SUFF(pVM);
12240 if (fIOString)
12241 {
12242#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12243 See @bugref{5752#c158}. Should work now. */
12244 /*
12245 * INS/OUTS - I/O String instruction.
12246 *
12247 * Use instruction-information if available, otherwise fall back on
12248 * interpreting the instruction.
12249 */
12250 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12251 fIOWrite ? 'w' : 'r'));
12252 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12253 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12254 {
12255 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12256 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12257 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12258 AssertRCReturn(rc2, rc2);
12259 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12260 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12261 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12262 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12263 if (fIOWrite)
12264 {
12265 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12266 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12267 }
12268 else
12269 {
12270 /*
12271 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12272 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12273 * See Intel Instruction spec. for "INS".
12274 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12275 */
12276 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12277 }
12278 }
12279 else
12280 {
12281 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12282 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12283 AssertRCReturn(rc2, rc2);
12284 rcStrict = IEMExecOne(pVCpu);
12285 }
12286 /** @todo IEM needs to be setting these flags somehow. */
12287 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12288 fUpdateRipAlready = true;
12289#else
12290 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12291 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12292 if (RT_SUCCESS(rcStrict))
12293 {
12294 if (fIOWrite)
12295 {
12296 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12297 (DISCPUMODE)pDis->uAddrMode, cbValue);
12298 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12299 }
12300 else
12301 {
12302 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12303 (DISCPUMODE)pDis->uAddrMode, cbValue);
12304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12305 }
12306 }
12307 else
12308 {
12309 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12310 pMixedCtx->rip));
12311 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12312 }
12313#endif
12314 }
12315 else
12316 {
12317 /*
12318 * IN/OUT - I/O instruction.
12319 */
12320 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12321 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12322 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12323 if (fIOWrite)
12324 {
12325 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12326 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12327 }
12328 else
12329 {
12330 uint32_t u32Result = 0;
12331 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12332 if (IOM_SUCCESS(rcStrict))
12333 {
12334 /* Save result of I/O IN instr. in AL/AX/EAX. */
12335 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12336 }
12337 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12338 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12340 }
12341 }
12342
12343 if (IOM_SUCCESS(rcStrict))
12344 {
12345 if (!fUpdateRipAlready)
12346 {
12347 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12348 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12349 }
12350
12351 /*
12352 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12353 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12354 */
12355 if (fIOString)
12356 {
12357 /** @todo Single-step for INS/OUTS with REP prefix? */
12358 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12359 }
12360 else if ( !fDbgStepping
12361 && fGstStepping)
12362 {
12363 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12364 }
12365
12366 /*
12367 * If any I/O breakpoints are armed, we need to check if one triggered
12368 * and take appropriate action.
12369 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12370 */
12371 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12372 AssertRCReturn(rc2, rc2);
12373
12374 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12375 * execution engines about whether hyper BPs and such are pending. */
12376 uint32_t const uDr7 = pMixedCtx->dr[7];
12377 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12378 && X86_DR7_ANY_RW_IO(uDr7)
12379 && (pMixedCtx->cr4 & X86_CR4_DE))
12380 || DBGFBpIsHwIoArmed(pVM)))
12381 {
12382 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12383
12384 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12385 VMMRZCallRing3Disable(pVCpu);
12386 HM_DISABLE_PREEMPT();
12387
12388 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12389
12390 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12391 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12392 {
12393 /* Raise #DB. */
12394 if (fIsGuestDbgActive)
12395 ASMSetDR6(pMixedCtx->dr[6]);
12396 if (pMixedCtx->dr[7] != uDr7)
12397 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12398
12399 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12400 }
12401 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12402 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12403 else if ( rcStrict2 != VINF_SUCCESS
12404 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12405 rcStrict = rcStrict2;
12406 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12407
12408 HM_RESTORE_PREEMPT();
12409 VMMRZCallRing3Enable(pVCpu);
12410 }
12411 }
12412
12413#ifdef VBOX_STRICT
12414 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12415 Assert(!fIOWrite);
12416 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12417 Assert(fIOWrite);
12418 else
12419 {
12420#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12421 * statuses, that the VMM device and some others may return. See
12422 * IOM_SUCCESS() for guidance. */
12423 AssertMsg( RT_FAILURE(rcStrict)
12424 || rcStrict == VINF_SUCCESS
12425 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12426 || rcStrict == VINF_EM_DBG_BREAKPOINT
12427 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12428 || rcStrict == VINF_EM_RAW_TO_R3
12429 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12430#endif
12431 }
12432#endif
12433
12434 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12435 return rcStrict;
12436}
12437
12438
12439/**
12440 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12441 * VM-exit.
12442 */
12443HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12444{
12445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12446
12447 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12448 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12449 AssertRCReturn(rc, rc);
12450 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12451 {
12452 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12453 AssertRCReturn(rc, rc);
12454 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12455 {
12456 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12457
12458 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12459 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12460
12461 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12462 Assert(!pVCpu->hm.s.Event.fPending);
12463 pVCpu->hm.s.Event.fPending = true;
12464 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12465 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12466 AssertRCReturn(rc, rc);
12467 if (fErrorCodeValid)
12468 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12469 else
12470 pVCpu->hm.s.Event.u32ErrCode = 0;
12471 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12472 && uVector == X86_XCPT_PF)
12473 {
12474 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12475 }
12476
12477 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12479 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12480 }
12481 }
12482
12483 /** @todo Emulate task switch someday, currently just going back to ring-3 for
12484 * emulation. */
12485 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12486 return VERR_EM_INTERPRETER;
12487}
12488
12489
12490/**
12491 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12492 */
12493HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12494{
12495 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12496 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12497 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12498 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12499 AssertRCReturn(rc, rc);
12500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12501 return VINF_EM_DBG_STEPPED;
12502}
12503
12504
12505/**
12506 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12507 */
12508HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12509{
12510 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12511
12512 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12513 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12514 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12515 { /* likely */ }
12516 else
12517 {
12518 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12519 rcStrict1 = VINF_SUCCESS;
12520 return rcStrict1;
12521 }
12522
12523#if 0
12524 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12525 * just sync the whole thing. */
12526 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12527#else
12528 /* Aggressive state sync. for now. */
12529 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12530 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12531 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12532#endif
12533 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12534 AssertRCReturn(rc, rc);
12535
12536 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12537 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12538 VBOXSTRICTRC rcStrict2;
12539 switch (uAccessType)
12540 {
12541 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12542 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12543 {
12544 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12545 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12546 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12547
12548 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12549 GCPhys &= PAGE_BASE_GC_MASK;
12550 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12551 PVM pVM = pVCpu->CTX_SUFF(pVM);
12552 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12553 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12554
12555 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12556 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12557 CPUMCTX2CORE(pMixedCtx), GCPhys);
12558 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12559 if ( rcStrict2 == VINF_SUCCESS
12560 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12561 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12562 {
12563 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12564 | HM_CHANGED_GUEST_RSP
12565 | HM_CHANGED_GUEST_RFLAGS
12566 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12567 rcStrict2 = VINF_SUCCESS;
12568 }
12569 break;
12570 }
12571
12572 default:
12573 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12574 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12575 break;
12576 }
12577
12578 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12579 if (rcStrict2 != VINF_SUCCESS)
12580 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12581 return rcStrict2;
12582}
12583
12584
12585/**
12586 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12587 * VM-exit.
12588 */
12589HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12590{
12591 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12592
12593 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12594 if (pVmxTransient->fWasGuestDebugStateActive)
12595 {
12596 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12597 HMVMX_RETURN_UNEXPECTED_EXIT();
12598 }
12599
12600 if ( !pVCpu->hm.s.fSingleInstruction
12601 && !pVmxTransient->fWasHyperDebugStateActive)
12602 {
12603 Assert(!DBGFIsStepping(pVCpu));
12604 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12605
12606 /* Don't intercept MOV DRx any more. */
12607 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12608 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12609 AssertRCReturn(rc, rc);
12610
12611 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12612 VMMRZCallRing3Disable(pVCpu);
12613 HM_DISABLE_PREEMPT();
12614
12615 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12616 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12617 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12618
12619 HM_RESTORE_PREEMPT();
12620 VMMRZCallRing3Enable(pVCpu);
12621
12622#ifdef VBOX_WITH_STATISTICS
12623 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12624 AssertRCReturn(rc, rc);
12625 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12627 else
12628 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12629#endif
12630 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12631 return VINF_SUCCESS;
12632 }
12633
12634 /*
12635 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12636 * Update the segment registers and DR7 from the CPU.
12637 */
12638 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12639 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12640 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12641 AssertRCReturn(rc, rc);
12642 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12643
12644 PVM pVM = pVCpu->CTX_SUFF(pVM);
12645 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12646 {
12647 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12648 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12649 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12650 if (RT_SUCCESS(rc))
12651 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12653 }
12654 else
12655 {
12656 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12657 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12658 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12659 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12660 }
12661
12662 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12663 if (RT_SUCCESS(rc))
12664 {
12665 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12666 AssertRCReturn(rc2, rc2);
12667 return VINF_SUCCESS;
12668 }
12669 return rc;
12670}
12671
12672
12673/**
12674 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12675 * Conditional VM-exit.
12676 */
12677HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12678{
12679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12680 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12681
12682 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12683 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12684 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12685 { /* likely */ }
12686 else
12687 {
12688 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12689 rcStrict1 = VINF_SUCCESS;
12690 return rcStrict1;
12691 }
12692
12693 RTGCPHYS GCPhys = 0;
12694 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12695
12696#if 0
12697 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12698#else
12699 /* Aggressive state sync. for now. */
12700 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12701 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12702 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12703#endif
12704 AssertRCReturn(rc, rc);
12705
12706 /*
12707 * If we succeed, resume guest execution.
12708 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12709 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12710 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12711 * weird case. See @bugref{6043}.
12712 */
12713 PVM pVM = pVCpu->CTX_SUFF(pVM);
12714 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12715 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12716 if ( rcStrict2 == VINF_SUCCESS
12717 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12718 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12719 {
12720 /* Successfully handled MMIO operation. */
12721 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12722 | HM_CHANGED_GUEST_RSP
12723 | HM_CHANGED_GUEST_RFLAGS
12724 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12725 return VINF_SUCCESS;
12726 }
12727 return rcStrict2;
12728}
12729
12730
12731/**
12732 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12733 * VM-exit.
12734 */
12735HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12736{
12737 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12738 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12739
12740 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12741 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12742 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12743 { /* likely */ }
12744 else
12745 {
12746 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12747 rcStrict1 = VINF_SUCCESS;
12748 return rcStrict1;
12749 }
12750
12751 RTGCPHYS GCPhys = 0;
12752 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12753 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12754#if 0
12755 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12756#else
12757 /* Aggressive state sync. for now. */
12758 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12759 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12760 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12761#endif
12762 AssertRCReturn(rc, rc);
12763
12764 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12765 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12766
12767 RTGCUINT uErrorCode = 0;
12768 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12769 uErrorCode |= X86_TRAP_PF_ID;
12770 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12771 uErrorCode |= X86_TRAP_PF_RW;
12772 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12773 uErrorCode |= X86_TRAP_PF_P;
12774
12775 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12776
12777 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12778 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12779
12780 /* Handle the pagefault trap for the nested shadow table. */
12781 PVM pVM = pVCpu->CTX_SUFF(pVM);
12782 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12783 TRPMResetTrap(pVCpu);
12784
12785 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12786 if ( rcStrict2 == VINF_SUCCESS
12787 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12788 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12789 {
12790 /* Successfully synced our nested page tables. */
12791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12793 | HM_CHANGED_GUEST_RSP
12794 | HM_CHANGED_GUEST_RFLAGS);
12795 return VINF_SUCCESS;
12796 }
12797
12798 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12799 return rcStrict2;
12800}
12801
12802/** @} */
12803
12804/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12805/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12806/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12807
12808/** @name VM-exit exception handlers.
12809 * @{
12810 */
12811
12812/**
12813 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12814 */
12815static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12816{
12817 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12818 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12819
12820 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12821 AssertRCReturn(rc, rc);
12822
12823 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12824 {
12825 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12826 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12827
12828 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12829 * provides VM-exit instruction length. If this causes problem later,
12830 * disassemble the instruction like it's done on AMD-V. */
12831 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12832 AssertRCReturn(rc2, rc2);
12833 return rc;
12834 }
12835
12836 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12837 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12838 return rc;
12839}
12840
12841
12842/**
12843 * VM-exit exception handler for \#BP (Breakpoint exception).
12844 */
12845static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12846{
12847 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12849
12850 /** @todo Try optimize this by not saving the entire guest state unless
12851 * really needed. */
12852 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12853 AssertRCReturn(rc, rc);
12854
12855 PVM pVM = pVCpu->CTX_SUFF(pVM);
12856 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12857 if (rc == VINF_EM_RAW_GUEST_TRAP)
12858 {
12859 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12860 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12861 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12862 AssertRCReturn(rc, rc);
12863
12864 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12865 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12866 }
12867
12868 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12869 return rc;
12870}
12871
12872
12873/**
12874 * VM-exit exception handler for \#AC (alignment check exception).
12875 */
12876static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12877{
12878 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12879
12880 /*
12881 * Re-inject it. We'll detect any nesting before getting here.
12882 */
12883 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12884 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12885 AssertRCReturn(rc, rc);
12886 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12887
12888 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12889 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12890 return VINF_SUCCESS;
12891}
12892
12893
12894/**
12895 * VM-exit exception handler for \#DB (Debug exception).
12896 */
12897static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12898{
12899 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12901 Log6(("XcptDB\n"));
12902
12903 /*
12904 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12905 * for processing.
12906 */
12907 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12908 AssertRCReturn(rc, rc);
12909
12910 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12911 uint64_t uDR6 = X86_DR6_INIT_VAL;
12912 uDR6 |= ( pVmxTransient->uExitQualification
12913 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12914
12915 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12916 if (rc == VINF_EM_RAW_GUEST_TRAP)
12917 {
12918 /*
12919 * The exception was for the guest. Update DR6, DR7.GD and
12920 * IA32_DEBUGCTL.LBR before forwarding it.
12921 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12922 */
12923 VMMRZCallRing3Disable(pVCpu);
12924 HM_DISABLE_PREEMPT();
12925
12926 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12927 pMixedCtx->dr[6] |= uDR6;
12928 if (CPUMIsGuestDebugStateActive(pVCpu))
12929 ASMSetDR6(pMixedCtx->dr[6]);
12930
12931 HM_RESTORE_PREEMPT();
12932 VMMRZCallRing3Enable(pVCpu);
12933
12934 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12935 AssertRCReturn(rc, rc);
12936
12937 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12938 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12939
12940 /* Paranoia. */
12941 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12942 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12943
12944 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12945 AssertRCReturn(rc, rc);
12946
12947 /*
12948 * Raise #DB in the guest.
12949 *
12950 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
12951 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
12952 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
12953 *
12954 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
12955 */
12956 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12957 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12958 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12959 AssertRCReturn(rc, rc);
12960 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12961 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12962 return VINF_SUCCESS;
12963 }
12964
12965 /*
12966 * Not a guest trap, must be a hypervisor related debug event then.
12967 * Update DR6 in case someone is interested in it.
12968 */
12969 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12970 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12971 CPUMSetHyperDR6(pVCpu, uDR6);
12972
12973 return rc;
12974}
12975
12976
12977/**
12978 * VM-exit exception handler for \#NM (Device-not-available exception: floating
12979 * point exception).
12980 */
12981static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12982{
12983 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12984
12985 /* We require CR0 and EFER. EFER is always up-to-date. */
12986 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12987 AssertRCReturn(rc, rc);
12988
12989 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
12990 VMMRZCallRing3Disable(pVCpu);
12991 HM_DISABLE_PREEMPT();
12992
12993 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
12994 if (pVmxTransient->fWasGuestFPUStateActive)
12995 {
12996 rc = VINF_EM_RAW_GUEST_TRAP;
12997 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
12998 }
12999 else
13000 {
13001#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13002 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13003#endif
13004 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13005 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13006 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13007 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13008 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13009 }
13010
13011 HM_RESTORE_PREEMPT();
13012 VMMRZCallRing3Enable(pVCpu);
13013
13014 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13015 {
13016 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13017 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13018 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13019 pVCpu->hm.s.fPreloadGuestFpu = true;
13020 }
13021 else
13022 {
13023 /* Forward #NM to the guest. */
13024 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13025 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13026 AssertRCReturn(rc, rc);
13027 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13028 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13029 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13030 }
13031
13032 return VINF_SUCCESS;
13033}
13034
13035
13036/**
13037 * VM-exit exception handler for \#GP (General-protection exception).
13038 *
13039 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13040 */
13041static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13042{
13043 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13045
13046 int rc;
13047 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13048 { /* likely */ }
13049 else
13050 {
13051#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13052 Assert(pVCpu->hm.s.fUsingDebugLoop);
13053#endif
13054 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13055 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13056 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13057 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13058 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13059 AssertRCReturn(rc, rc);
13060 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13061 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13062 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13063 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13064 return rc;
13065 }
13066
13067 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13068 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13069
13070 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13071 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13072 AssertRCReturn(rc, rc);
13073
13074 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13075 uint32_t cbOp = 0;
13076 PVM pVM = pVCpu->CTX_SUFF(pVM);
13077 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13078 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13079 if (RT_SUCCESS(rc))
13080 {
13081 rc = VINF_SUCCESS;
13082 Assert(cbOp == pDis->cbInstr);
13083 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13084 switch (pDis->pCurInstr->uOpcode)
13085 {
13086 case OP_CLI:
13087 {
13088 pMixedCtx->eflags.Bits.u1IF = 0;
13089 pMixedCtx->eflags.Bits.u1RF = 0;
13090 pMixedCtx->rip += pDis->cbInstr;
13091 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13092 if ( !fDbgStepping
13093 && pMixedCtx->eflags.Bits.u1TF)
13094 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13095 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13096 break;
13097 }
13098
13099 case OP_STI:
13100 {
13101 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13102 pMixedCtx->eflags.Bits.u1IF = 1;
13103 pMixedCtx->eflags.Bits.u1RF = 0;
13104 pMixedCtx->rip += pDis->cbInstr;
13105 if (!fOldIF)
13106 {
13107 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13108 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13109 }
13110 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13111 if ( !fDbgStepping
13112 && pMixedCtx->eflags.Bits.u1TF)
13113 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13115 break;
13116 }
13117
13118 case OP_HLT:
13119 {
13120 rc = VINF_EM_HALT;
13121 pMixedCtx->rip += pDis->cbInstr;
13122 pMixedCtx->eflags.Bits.u1RF = 0;
13123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13125 break;
13126 }
13127
13128 case OP_POPF:
13129 {
13130 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13131 uint32_t cbParm;
13132 uint32_t uMask;
13133 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13134 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13135 {
13136 cbParm = 4;
13137 uMask = 0xffffffff;
13138 }
13139 else
13140 {
13141 cbParm = 2;
13142 uMask = 0xffff;
13143 }
13144
13145 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13146 RTGCPTR GCPtrStack = 0;
13147 X86EFLAGS Eflags;
13148 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13149 &GCPtrStack);
13150 if (RT_SUCCESS(rc))
13151 {
13152 Assert(sizeof(Eflags.u32) >= cbParm);
13153 Eflags.u32 = 0;
13154 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13155 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13156 }
13157 if (RT_FAILURE(rc))
13158 {
13159 rc = VERR_EM_INTERPRETER;
13160 break;
13161 }
13162 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13163 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13164 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13165 pMixedCtx->esp += cbParm;
13166 pMixedCtx->esp &= uMask;
13167 pMixedCtx->rip += pDis->cbInstr;
13168 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13169 | HM_CHANGED_GUEST_RSP
13170 | HM_CHANGED_GUEST_RFLAGS);
13171 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13172 POPF restores EFLAGS.TF. */
13173 if ( !fDbgStepping
13174 && fGstStepping)
13175 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13177 break;
13178 }
13179
13180 case OP_PUSHF:
13181 {
13182 uint32_t cbParm;
13183 uint32_t uMask;
13184 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13185 {
13186 cbParm = 4;
13187 uMask = 0xffffffff;
13188 }
13189 else
13190 {
13191 cbParm = 2;
13192 uMask = 0xffff;
13193 }
13194
13195 /* Get the stack pointer & push the contents of eflags onto the stack. */
13196 RTGCPTR GCPtrStack = 0;
13197 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13198 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13199 if (RT_FAILURE(rc))
13200 {
13201 rc = VERR_EM_INTERPRETER;
13202 break;
13203 }
13204 X86EFLAGS Eflags = pMixedCtx->eflags;
13205 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13206 Eflags.Bits.u1RF = 0;
13207 Eflags.Bits.u1VM = 0;
13208
13209 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13210 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13211 {
13212 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13213 rc = VERR_EM_INTERPRETER;
13214 break;
13215 }
13216 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13217 pMixedCtx->esp -= cbParm;
13218 pMixedCtx->esp &= uMask;
13219 pMixedCtx->rip += pDis->cbInstr;
13220 pMixedCtx->eflags.Bits.u1RF = 0;
13221 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13222 | HM_CHANGED_GUEST_RSP
13223 | HM_CHANGED_GUEST_RFLAGS);
13224 if ( !fDbgStepping
13225 && pMixedCtx->eflags.Bits.u1TF)
13226 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13228 break;
13229 }
13230
13231 case OP_IRET:
13232 {
13233 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13234 * instruction reference. */
13235 RTGCPTR GCPtrStack = 0;
13236 uint32_t uMask = 0xffff;
13237 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13238 uint16_t aIretFrame[3];
13239 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13240 {
13241 rc = VERR_EM_INTERPRETER;
13242 break;
13243 }
13244 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13245 &GCPtrStack);
13246 if (RT_SUCCESS(rc))
13247 {
13248 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13249 PGMACCESSORIGIN_HM));
13250 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13251 }
13252 if (RT_FAILURE(rc))
13253 {
13254 rc = VERR_EM_INTERPRETER;
13255 break;
13256 }
13257 pMixedCtx->eip = 0;
13258 pMixedCtx->ip = aIretFrame[0];
13259 pMixedCtx->cs.Sel = aIretFrame[1];
13260 pMixedCtx->cs.ValidSel = aIretFrame[1];
13261 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13262 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13263 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13264 pMixedCtx->sp += sizeof(aIretFrame);
13265 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13266 | HM_CHANGED_GUEST_SEGMENT_REGS
13267 | HM_CHANGED_GUEST_RSP
13268 | HM_CHANGED_GUEST_RFLAGS);
13269 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13270 if ( !fDbgStepping
13271 && fGstStepping)
13272 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13273 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13274 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13275 break;
13276 }
13277
13278 case OP_INT:
13279 {
13280 uint16_t uVector = pDis->Param1.uValue & 0xff;
13281 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13282 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13283 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13284 break;
13285 }
13286
13287 case OP_INTO:
13288 {
13289 if (pMixedCtx->eflags.Bits.u1OF)
13290 {
13291 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13292 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13293 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13294 }
13295 else
13296 {
13297 pMixedCtx->eflags.Bits.u1RF = 0;
13298 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13299 }
13300 break;
13301 }
13302
13303 default:
13304 {
13305 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13306 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13307 EMCODETYPE_SUPERVISOR);
13308 rc = VBOXSTRICTRC_VAL(rc2);
13309 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13310 /** @todo We have to set pending-debug exceptions here when the guest is
13311 * single-stepping depending on the instruction that was interpreted. */
13312 Log4(("#GP rc=%Rrc\n", rc));
13313 break;
13314 }
13315 }
13316 }
13317 else
13318 rc = VERR_EM_INTERPRETER;
13319
13320 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13321 ("#GP Unexpected rc=%Rrc\n", rc));
13322 return rc;
13323}
13324
13325
13326/**
13327 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13328 * the exception reported in the VMX transient structure back into the VM.
13329 *
13330 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13331 * up-to-date.
13332 */
13333static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13334{
13335 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13336#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13337 Assert(pVCpu->hm.s.fUsingDebugLoop);
13338#endif
13339
13340 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13341 hmR0VmxCheckExitDueToEventDelivery(). */
13342 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13343 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13344 AssertRCReturn(rc, rc);
13345 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13346
13347#ifdef DEBUG_ramshankar
13348 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13349 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13350 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13351#endif
13352
13353 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13354 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13355 return VINF_SUCCESS;
13356}
13357
13358
13359/**
13360 * VM-exit exception handler for \#PF (Page-fault exception).
13361 */
13362static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13363{
13364 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13365 PVM pVM = pVCpu->CTX_SUFF(pVM);
13366 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13367 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13368 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13369 AssertRCReturn(rc, rc);
13370
13371 if (!pVM->hm.s.fNestedPaging)
13372 { /* likely */ }
13373 else
13374 {
13375#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13376 Assert(pVCpu->hm.s.fUsingDebugLoop);
13377#endif
13378 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13379 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13380 {
13381 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13382 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13383 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13384 }
13385 else
13386 {
13387 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13388 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13389 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13390 }
13391 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13392 return rc;
13393 }
13394
13395 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13396 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13397 if (pVmxTransient->fVectoringPF)
13398 {
13399 Assert(pVCpu->hm.s.Event.fPending);
13400 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13401 }
13402
13403 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13404 AssertRCReturn(rc, rc);
13405
13406 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13407 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13408
13409 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13410 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13411 (RTGCPTR)pVmxTransient->uExitQualification);
13412
13413 Log4(("#PF: rc=%Rrc\n", rc));
13414 if (rc == VINF_SUCCESS)
13415 {
13416 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13417 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13418 * memory? We don't update the whole state here... */
13419 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13420 | HM_CHANGED_GUEST_RSP
13421 | HM_CHANGED_GUEST_RFLAGS
13422 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13423 TRPMResetTrap(pVCpu);
13424 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13425 return rc;
13426 }
13427
13428 if (rc == VINF_EM_RAW_GUEST_TRAP)
13429 {
13430 if (!pVmxTransient->fVectoringDoublePF)
13431 {
13432 /* It's a guest page fault and needs to be reflected to the guest. */
13433 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13434 TRPMResetTrap(pVCpu);
13435 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13436 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13437 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13438 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13439 }
13440 else
13441 {
13442 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13443 TRPMResetTrap(pVCpu);
13444 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13445 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13446 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13447 }
13448
13449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13450 return VINF_SUCCESS;
13451 }
13452
13453 TRPMResetTrap(pVCpu);
13454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13455 return rc;
13456}
13457
13458/** @} */
13459
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette