VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 66756

Last change on this file since 66756 was 66756, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Forgot to translate soft. ints while converting IEM exception flags.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 597.9 KB
Line 
1/* $Id: HMVMXR0.cpp 66756 2017-05-03 12:13:03Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#define HMVMX_USE_IEM_EVENT_REFLECTION
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_SWAP_FPU_STATE
52# define HMVMX_ALWAYS_FLUSH_TLB
53# define HMVMX_ALWAYS_SWAP_EFER
54#endif
55
56
57/*********************************************************************************************************************************
58* Defined Constants And Macros *
59*********************************************************************************************************************************/
60/** Use the function table. */
61#define HMVMX_USE_FUNCTION_TABLE
62
63/** Determine which tagged-TLB flush handler to use. */
64#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
65#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
66#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
67#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
68
69/** @name Updated-guest-state flags.
70 * @{ */
71#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
72#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
73#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
74#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
75#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
76#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
77#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
78#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
79#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
80#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
81#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
82#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
83#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
84#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
85#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
86#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
87#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
88#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
89#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
90#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
91#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
92 | HMVMX_UPDATED_GUEST_RSP \
93 | HMVMX_UPDATED_GUEST_RFLAGS \
94 | HMVMX_UPDATED_GUEST_CR0 \
95 | HMVMX_UPDATED_GUEST_CR3 \
96 | HMVMX_UPDATED_GUEST_CR4 \
97 | HMVMX_UPDATED_GUEST_GDTR \
98 | HMVMX_UPDATED_GUEST_IDTR \
99 | HMVMX_UPDATED_GUEST_LDTR \
100 | HMVMX_UPDATED_GUEST_TR \
101 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
102 | HMVMX_UPDATED_GUEST_DR7 \
103 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
108 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109 | HMVMX_UPDATED_GUEST_INTR_STATE \
110 | HMVMX_UPDATED_GUEST_APIC_STATE)
111/** @} */
112
113/** @name
114 * Flags to skip redundant reads of some common VMCS fields that are not part of
115 * the guest-CPU state but are in the transient structure.
116 */
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
124/** @} */
125
126/** @name
127 * States of the VMCS.
128 *
129 * This does not reflect all possible VMCS states but currently only those
130 * needed for maintaining the VMCS consistently even when thread-context hooks
131 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
132 */
133#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
134#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
135#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
136/** @} */
137
138/**
139 * Exception bitmap mask for real-mode guests (real-on-v86).
140 *
141 * We need to intercept all exceptions manually except:
142 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
143 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
144 * due to bugs in Intel CPUs.
145 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
146 * support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*********************************************************************************************************************************
197* Structures and Typedefs *
198*********************************************************************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG fEFlags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit code qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u7Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 } ExitInstrInfo;
247 /** Whether the VM-entry failed or not. */
248 bool fVMEntryFailed;
249 /** Alignment. */
250 uint8_t abAlignment1[3];
251
252 /** The VM-entry interruption-information field. */
253 uint32_t uEntryIntInfo;
254 /** The VM-entry exception error code field. */
255 uint32_t uEntryXcptErrorCode;
256 /** The VM-entry instruction length field. */
257 uint32_t cbEntryInstr;
258
259 /** IDT-vectoring information field. */
260 uint32_t uIdtVectoringInfo;
261 /** IDT-vectoring error code. */
262 uint32_t uIdtVectoringErrorCode;
263
264 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
265 uint32_t fVmcsFieldsRead;
266
267 /** Whether the guest FPU was active at the time of VM-exit. */
268 bool fWasGuestFPUStateActive;
269 /** Whether the guest debug state was active at the time of VM-exit. */
270 bool fWasGuestDebugStateActive;
271 /** Whether the hyper debug state was active at the time of VM-exit. */
272 bool fWasHyperDebugStateActive;
273 /** Whether TSC-offsetting should be setup before VM-entry. */
274 bool fUpdateTscOffsettingAndPreemptTimer;
275 /** Whether the VM-exit was caused by a page-fault during delivery of a
276 * contributory exception or a page-fault. */
277 bool fVectoringDoublePF;
278 /** Whether the VM-exit was caused by a page-fault during delivery of an
279 * external interrupt or NMI. */
280 bool fVectoringPF;
281} VMXTRANSIENT;
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287/** Pointer to VMX transient state. */
288typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291/**
292 * MSR-bitmap read permissions.
293 */
294typedef enum VMXMSREXITREAD
295{
296 /** Reading this MSR causes a VM-exit. */
297 VMXMSREXIT_INTERCEPT_READ = 0xb,
298 /** Reading this MSR does not cause a VM-exit. */
299 VMXMSREXIT_PASSTHRU_READ
300} VMXMSREXITREAD;
301/** Pointer to MSR-bitmap read permissions. */
302typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304/**
305 * MSR-bitmap write permissions.
306 */
307typedef enum VMXMSREXITWRITE
308{
309 /** Writing to this MSR causes a VM-exit. */
310 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311 /** Writing to this MSR does not cause a VM-exit. */
312 VMXMSREXIT_PASSTHRU_WRITE
313} VMXMSREXITWRITE;
314/** Pointer to MSR-bitmap write permissions. */
315typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318/**
319 * VMX VM-exit handler.
320 *
321 * @returns Strict VBox status code (i.e. informational status codes too).
322 * @param pVCpu The cross context virtual CPU structure.
323 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
324 * out-of-sync. Make sure to update the required
325 * fields before using them.
326 * @param pVmxTransient Pointer to the VMX-transient structure.
327 */
328#ifndef HMVMX_USE_FUNCTION_TABLE
329typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330#else
331typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332/** Pointer to VM-exit handler. */
333typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334#endif
335
336/**
337 * VMX VM-exit handler, non-strict status code.
338 *
339 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
340 *
341 * @returns VBox status code, no informational status code returned.
342 * @param pVCpu The cross context virtual CPU structure.
343 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
344 * out-of-sync. Make sure to update the required
345 * fields before using them.
346 * @param pVmxTransient Pointer to the VMX-transient structure.
347 *
348 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
349 * use of that status code will be replaced with VINF_EM_SOMETHING
350 * later when switching over to IEM.
351 */
352#ifndef HMVMX_USE_FUNCTION_TABLE
353typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
354#else
355typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
356#endif
357
358
359/*********************************************************************************************************************************
360* Internal Functions *
361*********************************************************************************************************************************/
362static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
363static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
364static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
365static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
366 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
367 bool fStepping, uint32_t *puIntState);
368#if HC_ARCH_BITS == 32
369static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
370#endif
371#ifndef HMVMX_USE_FUNCTION_TABLE
372DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
373# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
374# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
375#else
376# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
377# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
378#endif
379
380
381/** @name VM-exit handlers.
382 * @{
383 */
384static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
385static FNVMXEXITHANDLER hmR0VmxExitExtInt;
386static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
393static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
394static FNVMXEXITHANDLER hmR0VmxExitCpuid;
395static FNVMXEXITHANDLER hmR0VmxExitGetsec;
396static FNVMXEXITHANDLER hmR0VmxExitHlt;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
398static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
399static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
400static FNVMXEXITHANDLER hmR0VmxExitVmcall;
401static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
404static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
405static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
406static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
407static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
408static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
412static FNVMXEXITHANDLER hmR0VmxExitMwait;
413static FNVMXEXITHANDLER hmR0VmxExitMtf;
414static FNVMXEXITHANDLER hmR0VmxExitMonitor;
415static FNVMXEXITHANDLER hmR0VmxExitPause;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
418static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
422static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
423static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
424static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
425static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
426static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
427static FNVMXEXITHANDLER hmR0VmxExitRdrand;
428static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
429/** @} */
430
431static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
440
441
442/*********************************************************************************************************************************
443* Global Variables *
444*********************************************************************************************************************************/
445#ifdef HMVMX_USE_FUNCTION_TABLE
446
447/**
448 * VMX_EXIT dispatch table.
449 */
450static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
451{
452 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
453 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
454 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
455 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
456 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
457 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
458 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
459 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
460 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
461 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
462 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
463 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
464 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
465 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
466 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
467 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
468 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
469 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
470 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
471 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
472 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
473 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
474 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
475 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
476 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
477 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
478 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
479 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
480 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
481 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
482 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
483 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
484 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
485 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
486 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
487 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
488 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
489 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
490 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
491 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
492 /* 40 UNDEFINED */ hmR0VmxExitPause,
493 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
494 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
495 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
496 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
497 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
498 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
501 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
502 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
503 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
504 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
505 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
506 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
507 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
508 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
509 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
510 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
511 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
512 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
513 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
514 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
515 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
516 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
517};
518#endif /* HMVMX_USE_FUNCTION_TABLE */
519
520#ifdef VBOX_STRICT
521static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
522{
523 /* 0 */ "(Not Used)",
524 /* 1 */ "VMCALL executed in VMX root operation.",
525 /* 2 */ "VMCLEAR with invalid physical address.",
526 /* 3 */ "VMCLEAR with VMXON pointer.",
527 /* 4 */ "VMLAUNCH with non-clear VMCS.",
528 /* 5 */ "VMRESUME with non-launched VMCS.",
529 /* 6 */ "VMRESUME after VMXOFF",
530 /* 7 */ "VM-entry with invalid control fields.",
531 /* 8 */ "VM-entry with invalid host state fields.",
532 /* 9 */ "VMPTRLD with invalid physical address.",
533 /* 10 */ "VMPTRLD with VMXON pointer.",
534 /* 11 */ "VMPTRLD with incorrect revision identifier.",
535 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
536 /* 13 */ "VMWRITE to read-only VMCS component.",
537 /* 14 */ "(Not Used)",
538 /* 15 */ "VMXON executed in VMX root operation.",
539 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
540 /* 17 */ "VM-entry with non-launched executing VMCS.",
541 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
542 /* 19 */ "VMCALL with non-clear VMCS.",
543 /* 20 */ "VMCALL with invalid VM-exit control fields.",
544 /* 21 */ "(Not Used)",
545 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
546 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
547 /* 24 */ "VMCALL with invalid SMM-monitor features.",
548 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
549 /* 26 */ "VM-entry with events blocked by MOV SS.",
550 /* 27 */ "(Not Used)",
551 /* 28 */ "Invalid operand to INVEPT/INVVPID."
552};
553#endif /* VBOX_STRICT */
554
555
556
557/**
558 * Updates the VM's last error record.
559 *
560 * If there was a VMX instruction error, reads the error data from the VMCS and
561 * updates VCPU's last error record as well.
562 *
563 * @param pVM The cross context VM structure.
564 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
565 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
566 * VERR_VMX_INVALID_VMCS_FIELD.
567 * @param rc The error code.
568 */
569static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
570{
571 AssertPtr(pVM);
572 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
573 || rc == VERR_VMX_UNABLE_TO_START_VM)
574 {
575 AssertPtrReturnVoid(pVCpu);
576 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
577 }
578 pVM->hm.s.lLastError = rc;
579}
580
581
582/**
583 * Reads the VM-entry interruption-information field from the VMCS into the VMX
584 * transient structure.
585 *
586 * @returns VBox status code.
587 * @param pVmxTransient Pointer to the VMX transient structure.
588 *
589 * @remarks No-long-jump zone!!!
590 */
591DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
592{
593 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
594 AssertRCReturn(rc, rc);
595 return VINF_SUCCESS;
596}
597
598
599#ifdef VBOX_STRICT
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615#endif /* VBOX_STRICT */
616
617
618#ifdef VBOX_STRICT
619/**
620 * Reads the VM-entry exception error code field from the VMCS into
621 * the VMX transient structure.
622 *
623 * @returns VBox status code.
624 * @param pVmxTransient Pointer to the VMX transient structure.
625 *
626 * @remarks No-long-jump zone!!!
627 */
628DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
629{
630 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
631 AssertRCReturn(rc, rc);
632 return VINF_SUCCESS;
633}
634#endif /* VBOX_STRICT */
635
636
637/**
638 * Reads the VM-exit interruption-information field from the VMCS into the VMX
639 * transient structure.
640 *
641 * @returns VBox status code.
642 * @param pVmxTransient Pointer to the VMX transient structure.
643 */
644DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
645{
646 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
647 {
648 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
649 AssertRCReturn(rc, rc);
650 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
651 }
652 return VINF_SUCCESS;
653}
654
655
656/**
657 * Reads the VM-exit interruption error code from the VMCS into the VMX
658 * transient structure.
659 *
660 * @returns VBox status code.
661 * @param pVmxTransient Pointer to the VMX transient structure.
662 */
663DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
664{
665 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
666 {
667 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
668 AssertRCReturn(rc, rc);
669 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
670 }
671 return VINF_SUCCESS;
672}
673
674
675/**
676 * Reads the VM-exit instruction length field from the VMCS into the VMX
677 * transient structure.
678 *
679 * @returns VBox status code.
680 * @param pVmxTransient Pointer to the VMX transient structure.
681 */
682DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the VM-exit instruction-information field from the VMCS into
696 * the VMX transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Reads the exit code qualification from the VMCS into the VMX transient
715 * structure.
716 *
717 * @returns VBox status code.
718 * @param pVCpu The cross context virtual CPU structure of the
719 * calling EMT. (Required for the VMCS cache case.)
720 * @param pVmxTransient Pointer to the VMX transient structure.
721 */
722DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
723{
724 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
725 {
726 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
727 AssertRCReturn(rc, rc);
728 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
729 }
730 return VINF_SUCCESS;
731}
732
733
734/**
735 * Reads the IDT-vectoring information field from the VMCS into the VMX
736 * transient structure.
737 *
738 * @returns VBox status code.
739 * @param pVmxTransient Pointer to the VMX transient structure.
740 *
741 * @remarks No-long-jump zone!!!
742 */
743DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
744{
745 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
746 {
747 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
748 AssertRCReturn(rc, rc);
749 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
750 }
751 return VINF_SUCCESS;
752}
753
754
755/**
756 * Reads the IDT-vectoring error code from the VMCS into the VMX
757 * transient structure.
758 *
759 * @returns VBox status code.
760 * @param pVmxTransient Pointer to the VMX transient structure.
761 */
762DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
763{
764 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
765 {
766 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
767 AssertRCReturn(rc, rc);
768 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
769 }
770 return VINF_SUCCESS;
771}
772
773
774/**
775 * Enters VMX root mode operation on the current CPU.
776 *
777 * @returns VBox status code.
778 * @param pVM The cross context VM structure. Can be
779 * NULL, after a resume.
780 * @param HCPhysCpuPage Physical address of the VMXON region.
781 * @param pvCpuPage Pointer to the VMXON region.
782 */
783static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
784{
785 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
786 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
787 Assert(pvCpuPage);
788 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
789
790 if (pVM)
791 {
792 /* Write the VMCS revision dword to the VMXON region. */
793 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
794 }
795
796 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
797 RTCCUINTREG fEFlags = ASMIntDisableFlags();
798
799 /* Enable the VMX bit in CR4 if necessary. */
800 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
801
802 /* Enter VMX root mode. */
803 int rc = VMXEnable(HCPhysCpuPage);
804 if (RT_FAILURE(rc))
805 {
806 if (!(uOldCr4 & X86_CR4_VMXE))
807 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
808
809 if (pVM)
810 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
811 }
812
813 /* Restore interrupts. */
814 ASMSetFlags(fEFlags);
815 return rc;
816}
817
818
819/**
820 * Exits VMX root mode operation on the current CPU.
821 *
822 * @returns VBox status code.
823 */
824static int hmR0VmxLeaveRootMode(void)
825{
826 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
827
828 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
829 RTCCUINTREG fEFlags = ASMIntDisableFlags();
830
831 /* If we're for some reason not in VMX root mode, then don't leave it. */
832 RTCCUINTREG uHostCR4 = ASMGetCR4();
833
834 int rc;
835 if (uHostCR4 & X86_CR4_VMXE)
836 {
837 /* Exit VMX root mode and clear the VMX bit in CR4. */
838 VMXDisable();
839 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
840 rc = VINF_SUCCESS;
841 }
842 else
843 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
844
845 /* Restore interrupts. */
846 ASMSetFlags(fEFlags);
847 return rc;
848}
849
850
851/**
852 * Allocates and maps one physically contiguous page. The allocated page is
853 * zero'd out. (Used by various VT-x structures).
854 *
855 * @returns IPRT status code.
856 * @param pMemObj Pointer to the ring-0 memory object.
857 * @param ppVirt Where to store the virtual address of the
858 * allocation.
859 * @param pHCPhys Where to store the physical address of the
860 * allocation.
861 */
862DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
863{
864 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
866 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
867
868 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
869 if (RT_FAILURE(rc))
870 return rc;
871 *ppVirt = RTR0MemObjAddress(*pMemObj);
872 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
873 ASMMemZero32(*ppVirt, PAGE_SIZE);
874 return VINF_SUCCESS;
875}
876
877
878/**
879 * Frees and unmaps an allocated physical page.
880 *
881 * @param pMemObj Pointer to the ring-0 memory object.
882 * @param ppVirt Where to re-initialize the virtual address of
883 * allocation as 0.
884 * @param pHCPhys Where to re-initialize the physical address of the
885 * allocation as 0.
886 */
887DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
888{
889 AssertPtr(pMemObj);
890 AssertPtr(ppVirt);
891 AssertPtr(pHCPhys);
892 if (*pMemObj != NIL_RTR0MEMOBJ)
893 {
894 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
895 AssertRC(rc);
896 *pMemObj = NIL_RTR0MEMOBJ;
897 *ppVirt = 0;
898 *pHCPhys = 0;
899 }
900}
901
902
903/**
904 * Worker function to free VT-x related structures.
905 *
906 * @returns IPRT status code.
907 * @param pVM The cross context VM structure.
908 */
909static void hmR0VmxStructsFree(PVM pVM)
910{
911 for (VMCPUID i = 0; i < pVM->cCpus; i++)
912 {
913 PVMCPU pVCpu = &pVM->aCpus[i];
914 AssertPtr(pVCpu);
915
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
918
919 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
921
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923 }
924
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928#endif
929}
930
931
932/**
933 * Worker function to allocate VT-x related VM structures.
934 *
935 * @returns IPRT status code.
936 * @param pVM The cross context VM structure.
937 */
938static int hmR0VmxStructsAlloc(PVM pVM)
939{
940 /*
941 * Initialize members up-front so we can cleanup properly on allocation failure.
942 */
943#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955#endif
956 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959 for (VMCPUID i = 0; i < pVM->cCpus; i++)
960 {
961 PVMCPU pVCpu = &pVM->aCpus[i];
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
965 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
966 }
967#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
968#undef VMXLOCAL_INIT_VM_MEMOBJ
969
970 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
971 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
972 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
973 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
974
975 /*
976 * Allocate all the VT-x structures.
977 */
978 int rc = VINF_SUCCESS;
979#ifdef VBOX_WITH_CRASHDUMP_MAGIC
980 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
981 if (RT_FAILURE(rc))
982 goto cleanup;
983 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
984 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
985#endif
986
987 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
988 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
989 {
990 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
991 &pVM->hm.s.vmx.HCPhysApicAccess);
992 if (RT_FAILURE(rc))
993 goto cleanup;
994 }
995
996 /*
997 * Initialize per-VCPU VT-x structures.
998 */
999 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1000 {
1001 PVMCPU pVCpu = &pVM->aCpus[i];
1002 AssertPtr(pVCpu);
1003
1004 /* Allocate the VM control structure (VMCS). */
1005 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1006 if (RT_FAILURE(rc))
1007 goto cleanup;
1008
1009 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1010 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1112 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1113 */
1114 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1115 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1116 {
1117 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1118 pCpu->fFlushAsidBeforeUse = false;
1119 }
1120 else
1121 pCpu->fFlushAsidBeforeUse = true;
1122
1123 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1124 ++pCpu->cTlbFlushes;
1125
1126 return VINF_SUCCESS;
1127}
1128
1129
1130/**
1131 * Deactivates VT-x on the current CPU.
1132 *
1133 * @returns VBox status code.
1134 * @param pCpu Pointer to the global CPU info struct.
1135 * @param pvCpuPage Pointer to the VMXON region.
1136 * @param HCPhysCpuPage Physical address of the VMXON region.
1137 *
1138 * @remarks This function should never be called when SUPR0EnableVTx() or
1139 * similar was used to enable VT-x on the host.
1140 */
1141VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1142{
1143 NOREF(pCpu);
1144 NOREF(pvCpuPage);
1145 NOREF(HCPhysCpuPage);
1146
1147 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1148 return hmR0VmxLeaveRootMode();
1149}
1150
1151
1152/**
1153 * Sets the permission bits for the specified MSR in the MSR bitmap.
1154 *
1155 * @param pVCpu The cross context virtual CPU structure.
1156 * @param uMsr The MSR value.
1157 * @param enmRead Whether reading this MSR causes a VM-exit.
1158 * @param enmWrite Whether writing this MSR causes a VM-exit.
1159 */
1160static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1161{
1162 int32_t iBit;
1163 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1164
1165 /*
1166 * Layout:
1167 * 0x000 - 0x3ff - Low MSR read bits
1168 * 0x400 - 0x7ff - High MSR read bits
1169 * 0x800 - 0xbff - Low MSR write bits
1170 * 0xc00 - 0xfff - High MSR write bits
1171 */
1172 if (uMsr <= 0x00001FFF)
1173 iBit = uMsr;
1174 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1175 {
1176 iBit = uMsr - UINT32_C(0xC0000000);
1177 pbMsrBitmap += 0x400;
1178 }
1179 else
1180 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1181
1182 Assert(iBit <= 0x1fff);
1183 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1184 ASMBitSet(pbMsrBitmap, iBit);
1185 else
1186 ASMBitClear(pbMsrBitmap, iBit);
1187
1188 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1189 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1190 else
1191 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1192}
1193
1194
1195#ifdef VBOX_STRICT
1196/**
1197 * Gets the permission bits for the specified MSR in the MSR bitmap.
1198 *
1199 * @returns VBox status code.
1200 * @retval VINF_SUCCESS if the specified MSR is found.
1201 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1202 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1203 *
1204 * @param pVCpu The cross context virtual CPU structure.
1205 * @param uMsr The MSR.
1206 * @param penmRead Where to store the read permissions.
1207 * @param penmWrite Where to store the write permissions.
1208 */
1209static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1210{
1211 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1212 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1213 int32_t iBit;
1214 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1215
1216 /* See hmR0VmxSetMsrPermission() for the layout. */
1217 if (uMsr <= 0x00001FFF)
1218 iBit = uMsr;
1219 else if ( uMsr >= 0xC0000000
1220 && uMsr <= 0xC0001FFF)
1221 {
1222 iBit = (uMsr - 0xC0000000);
1223 pbMsrBitmap += 0x400;
1224 }
1225 else
1226 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1227
1228 Assert(iBit <= 0x1fff);
1229 if (ASMBitTest(pbMsrBitmap, iBit))
1230 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1231 else
1232 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1233
1234 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1235 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1236 else
1237 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1238 return VINF_SUCCESS;
1239}
1240#endif /* VBOX_STRICT */
1241
1242
1243/**
1244 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1245 * area.
1246 *
1247 * @returns VBox status code.
1248 * @param pVCpu The cross context virtual CPU structure.
1249 * @param cMsrs The number of MSRs.
1250 */
1251DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1252{
1253 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1254 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1255 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256 {
1257 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260 }
1261
1262 /* Update number of guest MSRs to load/store across the world-switch. */
1263 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268 AssertRCReturn(rc, rc);
1269
1270 /* Update the VCPU's copy of the MSR count. */
1271 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273 return VINF_SUCCESS;
1274}
1275
1276
1277/**
1278 * Adds a new (or updates the value of an existing) guest/host MSR
1279 * pair to be swapped during the world-switch as part of the
1280 * auto-load/store MSR area in the VMCS.
1281 *
1282 * @returns VBox status code.
1283 * @param pVCpu The cross context virtual CPU structure.
1284 * @param uMsr The MSR.
1285 * @param uGuestMsrValue Value of the guest MSR.
1286 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1287 * necessary.
1288 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289 * its value was updated. Optional, can be NULL.
1290 */
1291static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292 bool *pfAddedAndUpdated)
1293{
1294 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296 uint32_t i;
1297 for (i = 0; i < cMsrs; i++)
1298 {
1299 if (pGuestMsr->u32Msr == uMsr)
1300 break;
1301 pGuestMsr++;
1302 }
1303
1304 bool fAdded = false;
1305 if (i == cMsrs)
1306 {
1307 ++cMsrs;
1308 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1313 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315 fAdded = true;
1316 }
1317
1318 /* Update the MSR values in the auto-load/store MSR area. */
1319 pGuestMsr->u32Msr = uMsr;
1320 pGuestMsr->u64Value = uGuestMsrValue;
1321
1322 /* Create/update the MSR slot in the host MSR area. */
1323 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324 pHostMsr += i;
1325 pHostMsr->u32Msr = uMsr;
1326
1327 /*
1328 * Update the host MSR only when requested by the caller AND when we're
1329 * adding it to the auto-load/store area. Otherwise, it would have been
1330 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1331 */
1332 bool fUpdatedMsrValue = false;
1333 if ( fAdded
1334 && fUpdateHostMsr)
1335 {
1336 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339 fUpdatedMsrValue = true;
1340 }
1341
1342 if (pfAddedAndUpdated)
1343 *pfAddedAndUpdated = fUpdatedMsrValue;
1344 return VINF_SUCCESS;
1345}
1346
1347
1348/**
1349 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1350 * auto-load/store MSR area in the VMCS.
1351 *
1352 * @returns VBox status code.
1353 * @param pVCpu The cross context virtual CPU structure.
1354 * @param uMsr The MSR.
1355 */
1356static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357{
1358 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360 for (uint32_t i = 0; i < cMsrs; i++)
1361 {
1362 /* Find the MSR. */
1363 if (pGuestMsr->u32Msr == uMsr)
1364 {
1365 /* If it's the last MSR, simply reduce the count. */
1366 if (i == cMsrs - 1)
1367 {
1368 --cMsrs;
1369 break;
1370 }
1371
1372 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374 pLastGuestMsr += cMsrs - 1;
1375 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 pLastHostMsr += cMsrs - 1;
1381 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382 pHostMsr->u64Value = pLastHostMsr->u64Value;
1383 --cMsrs;
1384 break;
1385 }
1386 pGuestMsr++;
1387 }
1388
1389 /* Update the VMCS if the count changed (meaning the MSR was found). */
1390 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391 {
1392 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393 AssertRCReturn(rc, rc);
1394
1395 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1397 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400 return VINF_SUCCESS;
1401 }
1402
1403 return VERR_NOT_FOUND;
1404}
1405
1406
1407/**
1408 * Checks if the specified guest MSR is part of the auto-load/store area in
1409 * the VMCS.
1410 *
1411 * @returns true if found, false otherwise.
1412 * @param pVCpu The cross context virtual CPU structure.
1413 * @param uMsr The MSR to find.
1414 */
1415static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416{
1417 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421 {
1422 if (pGuestMsr->u32Msr == uMsr)
1423 return true;
1424 }
1425 return false;
1426}
1427
1428
1429/**
1430 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431 *
1432 * @param pVCpu The cross context virtual CPU structure.
1433 *
1434 * @remarks No-long-jump zone!!!
1435 */
1436static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437{
1438 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444 {
1445 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447 /*
1448 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450 */
1451 if (pHostMsr->u32Msr == MSR_K6_EFER)
1452 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453 else
1454 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455 }
1456
1457 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458}
1459
1460
1461/**
1462 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1463 * perform lazy restoration of the host MSRs while leaving VT-x.
1464 *
1465 * @param pVCpu The cross context virtual CPU structure.
1466 *
1467 * @remarks No-long-jump zone!!!
1468 */
1469static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470{
1471 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473 /*
1474 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475 */
1476 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477 {
1478 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479#if HC_ARCH_BITS == 64
1480 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481 {
1482 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486 }
1487#endif
1488 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1489 }
1490}
1491
1492
1493/**
1494 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1495 * lazily while leaving VT-x.
1496 *
1497 * @returns true if it does, false otherwise.
1498 * @param pVCpu The cross context virtual CPU structure.
1499 * @param uMsr The MSR to check.
1500 */
1501static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502{
1503 NOREF(pVCpu);
1504#if HC_ARCH_BITS == 64
1505 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506 {
1507 switch (uMsr)
1508 {
1509 case MSR_K8_LSTAR:
1510 case MSR_K6_STAR:
1511 case MSR_K8_SF_MASK:
1512 case MSR_K8_KERNEL_GS_BASE:
1513 return true;
1514 }
1515 }
1516#else
1517 RT_NOREF(pVCpu, uMsr);
1518#endif
1519 return false;
1520}
1521
1522
1523/**
1524 * Saves a set of guest MSRs back into the guest-CPU context.
1525 *
1526 * @param pVCpu The cross context virtual CPU structure.
1527 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1528 * out-of-sync. Make sure to update the required fields
1529 * before using them.
1530 *
1531 * @remarks No-long-jump zone!!!
1532 */
1533static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1534{
1535 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1536 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1537
1538 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1539 {
1540 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541#if HC_ARCH_BITS == 64
1542 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543 {
1544 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1545 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1546 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1547 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1548 }
1549#else
1550 NOREF(pMixedCtx);
1551#endif
1552 }
1553}
1554
1555
1556/**
1557 * Loads a set of guests MSRs to allow read/passthru to the guest.
1558 *
1559 * The name of this function is slightly confusing. This function does NOT
1560 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1561 * common prefix for functions dealing with "lazy restoration" of the shared
1562 * MSRs.
1563 *
1564 * @param pVCpu The cross context virtual CPU structure.
1565 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1566 * out-of-sync. Make sure to update the required fields
1567 * before using them.
1568 *
1569 * @remarks No-long-jump zone!!!
1570 */
1571static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1572{
1573 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1574 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1575
1576 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1577#if HC_ARCH_BITS == 64
1578 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1579 {
1580 /*
1581 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1582 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1583 * we can skip a few MSR writes.
1584 *
1585 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1586 * guest MSR values in the guest-CPU context might be different to what's currently
1587 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1588 * CPU, see @bugref{8728}.
1589 */
1590 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1591 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1592 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1593 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1594 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1595 {
1596#ifdef VBOX_STRICT
1597 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1598 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1599 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1600 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1601#endif
1602 }
1603 else
1604 {
1605 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1606 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1607 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1608 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1609 }
1610 }
1611#else
1612 RT_NOREF(pMixedCtx);
1613#endif
1614 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1615}
1616
1617
1618/**
1619 * Performs lazy restoration of the set of host MSRs if they were previously
1620 * loaded with guest MSR values.
1621 *
1622 * @param pVCpu The cross context virtual CPU structure.
1623 *
1624 * @remarks No-long-jump zone!!!
1625 * @remarks The guest MSRs should have been saved back into the guest-CPU
1626 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1627 */
1628static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1629{
1630 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1632
1633 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1634 {
1635 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1636#if HC_ARCH_BITS == 64
1637 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1638 {
1639 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1640 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1641 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1642 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1643 }
1644#endif
1645 }
1646 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1647}
1648
1649
1650/**
1651 * Verifies that our cached values of the VMCS controls are all
1652 * consistent with what's actually present in the VMCS.
1653 *
1654 * @returns VBox status code.
1655 * @param pVCpu The cross context virtual CPU structure.
1656 */
1657static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1658{
1659 uint32_t u32Val;
1660 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1661 AssertRCReturn(rc, rc);
1662 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1663 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1666 AssertRCReturn(rc, rc);
1667 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1668 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1669
1670 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1671 AssertRCReturn(rc, rc);
1672 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1673 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1674
1675 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1676 AssertRCReturn(rc, rc);
1677 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1678 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1679
1680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681 {
1682 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1683 AssertRCReturn(rc, rc);
1684 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1685 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1686 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1687 }
1688
1689 return VINF_SUCCESS;
1690}
1691
1692
1693#ifdef VBOX_STRICT
1694/**
1695 * Verifies that our cached host EFER value has not changed
1696 * since we cached it.
1697 *
1698 * @param pVCpu The cross context virtual CPU structure.
1699 */
1700static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1701{
1702 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1703
1704 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1705 {
1706 uint64_t u64Val;
1707 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1708 AssertRC(rc);
1709
1710 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1711 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1712 }
1713}
1714
1715
1716/**
1717 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1718 * VMCS are correct.
1719 *
1720 * @param pVCpu The cross context virtual CPU structure.
1721 */
1722static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1723{
1724 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1725
1726 /* Verify MSR counts in the VMCS are what we think it should be. */
1727 uint32_t cMsrs;
1728 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1729 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1730
1731 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1732 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1733
1734 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1735 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1736
1737 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1738 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1739 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1740 {
1741 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1742 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1743 pGuestMsr->u32Msr, cMsrs));
1744
1745 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1746 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1747 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1748
1749 /* Verify that the permissions are as expected in the MSR bitmap. */
1750 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1751 {
1752 VMXMSREXITREAD enmRead;
1753 VMXMSREXITWRITE enmWrite;
1754 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1755 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1756 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1757 {
1758 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1759 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1760 }
1761 else
1762 {
1763 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1764 pGuestMsr->u32Msr, cMsrs));
1765 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1766 pGuestMsr->u32Msr, cMsrs));
1767 }
1768 }
1769 }
1770}
1771#endif /* VBOX_STRICT */
1772
1773
1774/**
1775 * Flushes the TLB using EPT.
1776 *
1777 * @returns VBox status code.
1778 * @param pVCpu The cross context virtual CPU structure of the calling
1779 * EMT. Can be NULL depending on @a enmFlush.
1780 * @param enmFlush Type of flush.
1781 *
1782 * @remarks Caller is responsible for making sure this function is called only
1783 * when NestedPaging is supported and providing @a enmFlush that is
1784 * supported by the CPU.
1785 * @remarks Can be called with interrupts disabled.
1786 */
1787static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1788{
1789 uint64_t au64Descriptor[2];
1790 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1791 au64Descriptor[0] = 0;
1792 else
1793 {
1794 Assert(pVCpu);
1795 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1796 }
1797 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1798
1799 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1800 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1801 rc));
1802 if ( RT_SUCCESS(rc)
1803 && pVCpu)
1804 {
1805 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1806 }
1807}
1808
1809
1810/**
1811 * Flushes the TLB using VPID.
1812 *
1813 * @returns VBox status code.
1814 * @param pVM The cross context VM structure.
1815 * @param pVCpu The cross context virtual CPU structure of the calling
1816 * EMT. Can be NULL depending on @a enmFlush.
1817 * @param enmFlush Type of flush.
1818 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1819 * on @a enmFlush).
1820 *
1821 * @remarks Can be called with interrupts disabled.
1822 */
1823static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1824{
1825 NOREF(pVM);
1826 AssertPtr(pVM);
1827 Assert(pVM->hm.s.vmx.fVpid);
1828
1829 uint64_t au64Descriptor[2];
1830 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1831 {
1832 au64Descriptor[0] = 0;
1833 au64Descriptor[1] = 0;
1834 }
1835 else
1836 {
1837 AssertPtr(pVCpu);
1838 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1839 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1840 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1841 au64Descriptor[1] = GCPtr;
1842 }
1843
1844 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1845 AssertMsg(rc == VINF_SUCCESS,
1846 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1847 if ( RT_SUCCESS(rc)
1848 && pVCpu)
1849 {
1850 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1851 }
1852}
1853
1854
1855/**
1856 * Invalidates a guest page by guest virtual address. Only relevant for
1857 * EPT/VPID, otherwise there is nothing really to invalidate.
1858 *
1859 * @returns VBox status code.
1860 * @param pVM The cross context VM structure.
1861 * @param pVCpu The cross context virtual CPU structure.
1862 * @param GCVirt Guest virtual address of the page to invalidate.
1863 */
1864VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1865{
1866 AssertPtr(pVM);
1867 AssertPtr(pVCpu);
1868 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1869
1870 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1871 if (!fFlushPending)
1872 {
1873 /*
1874 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1875 * See @bugref{6043} and @bugref{6177}.
1876 *
1877 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1878 * function maybe called in a loop with individual addresses.
1879 */
1880 if (pVM->hm.s.vmx.fVpid)
1881 {
1882 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1883 {
1884 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1885 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1886 }
1887 else
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890 else if (pVM->hm.s.fNestedPaging)
1891 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1892 }
1893
1894 return VINF_SUCCESS;
1895}
1896
1897
1898/**
1899 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1900 * otherwise there is nothing really to invalidate.
1901 *
1902 * @returns VBox status code.
1903 * @param pVM The cross context VM structure.
1904 * @param pVCpu The cross context virtual CPU structure.
1905 * @param GCPhys Guest physical address of the page to invalidate.
1906 */
1907VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1908{
1909 NOREF(pVM); NOREF(GCPhys);
1910 LogFlowFunc(("%RGp\n", GCPhys));
1911
1912 /*
1913 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1914 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1915 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1916 */
1917 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1918 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1919 return VINF_SUCCESS;
1920}
1921
1922
1923/**
1924 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1925 * case where neither EPT nor VPID is supported by the CPU.
1926 *
1927 * @param pVM The cross context VM structure.
1928 * @param pVCpu The cross context virtual CPU structure.
1929 * @param pCpu Pointer to the global HM struct.
1930 *
1931 * @remarks Called with interrupts disabled.
1932 */
1933static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1934{
1935 AssertPtr(pVCpu);
1936 AssertPtr(pCpu);
1937 NOREF(pVM);
1938
1939 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1940
1941 Assert(pCpu->idCpu != NIL_RTCPUID);
1942 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1943 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1944 pVCpu->hm.s.fForceTLBFlush = false;
1945 return;
1946}
1947
1948
1949/**
1950 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1951 *
1952 * @param pVM The cross context VM structure.
1953 * @param pVCpu The cross context virtual CPU structure.
1954 * @param pCpu Pointer to the global HM CPU struct.
1955 * @remarks All references to "ASID" in this function pertains to "VPID" in
1956 * Intel's nomenclature. The reason is, to avoid confusion in compare
1957 * statements since the host-CPU copies are named "ASID".
1958 *
1959 * @remarks Called with interrupts disabled.
1960 */
1961static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1962{
1963#ifdef VBOX_WITH_STATISTICS
1964 bool fTlbFlushed = false;
1965# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1966# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1967 if (!fTlbFlushed) \
1968 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1969 } while (0)
1970#else
1971# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1972# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1973#endif
1974
1975 AssertPtr(pVM);
1976 AssertPtr(pCpu);
1977 AssertPtr(pVCpu);
1978 Assert(pCpu->idCpu != NIL_RTCPUID);
1979
1980 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1981 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1982 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1983
1984 /*
1985 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1986 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1987 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1988 */
1989 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1990 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1991 {
1992 ++pCpu->uCurrentAsid;
1993 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1994 {
1995 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1996 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1997 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1998 }
1999
2000 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2001 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2002 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2003
2004 /*
2005 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2006 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2007 */
2008 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2009 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2010 HMVMX_SET_TAGGED_TLB_FLUSHED();
2011 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2012 }
2013
2014 /* Check for explicit TLB flushes. */
2015 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2016 {
2017 /*
2018 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2019 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2020 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2021 * but not guest-physical mappings.
2022 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2023 */
2024 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2026 HMVMX_SET_TAGGED_TLB_FLUSHED();
2027 }
2028
2029 pVCpu->hm.s.fForceTLBFlush = false;
2030 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2031
2032 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2033 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2034 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2035 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2036 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2037 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2038 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2039 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2040 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2041
2042 /* Update VMCS with the VPID. */
2043 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2044 AssertRC(rc);
2045
2046#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2047}
2048
2049
2050/**
2051 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2052 *
2053 * @returns VBox status code.
2054 * @param pVM The cross context VM structure.
2055 * @param pVCpu The cross context virtual CPU structure.
2056 * @param pCpu Pointer to the global HM CPU struct.
2057 *
2058 * @remarks Called with interrupts disabled.
2059 */
2060static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2061{
2062 AssertPtr(pVM);
2063 AssertPtr(pVCpu);
2064 AssertPtr(pCpu);
2065 Assert(pCpu->idCpu != NIL_RTCPUID);
2066 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2067 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2068
2069 /*
2070 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2071 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2072 */
2073 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2074 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2075 {
2076 pVCpu->hm.s.fForceTLBFlush = true;
2077 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2078 }
2079
2080 /* Check for explicit TLB flushes. */
2081 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2082 {
2083 pVCpu->hm.s.fForceTLBFlush = true;
2084 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2085 }
2086
2087 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2088 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2089
2090 if (pVCpu->hm.s.fForceTLBFlush)
2091 {
2092 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2093 pVCpu->hm.s.fForceTLBFlush = false;
2094 }
2095}
2096
2097
2098/**
2099 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2100 *
2101 * @returns VBox status code.
2102 * @param pVM The cross context VM structure.
2103 * @param pVCpu The cross context virtual CPU structure.
2104 * @param pCpu Pointer to the global HM CPU struct.
2105 *
2106 * @remarks Called with interrupts disabled.
2107 */
2108static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2109{
2110 AssertPtr(pVM);
2111 AssertPtr(pVCpu);
2112 AssertPtr(pCpu);
2113 Assert(pCpu->idCpu != NIL_RTCPUID);
2114 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2115 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2116
2117 /*
2118 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2119 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2120 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2121 */
2122 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2123 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2124 {
2125 pVCpu->hm.s.fForceTLBFlush = true;
2126 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2127 }
2128
2129 /* Check for explicit TLB flushes. */
2130 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2131 {
2132 /*
2133 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2134 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2135 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2136 */
2137 pVCpu->hm.s.fForceTLBFlush = true;
2138 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2139 }
2140
2141 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2142 if (pVCpu->hm.s.fForceTLBFlush)
2143 {
2144 ++pCpu->uCurrentAsid;
2145 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2146 {
2147 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2148 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2149 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2150 }
2151
2152 pVCpu->hm.s.fForceTLBFlush = false;
2153 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2154 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2155 if (pCpu->fFlushAsidBeforeUse)
2156 {
2157 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2158 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2159 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2160 {
2161 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2162 pCpu->fFlushAsidBeforeUse = false;
2163 }
2164 else
2165 {
2166 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2167 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2168 }
2169 }
2170 }
2171
2172 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2173 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2174 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2175 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2176 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2177 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2178 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2179
2180 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2181 AssertRC(rc);
2182}
2183
2184
2185/**
2186 * Flushes the guest TLB entry based on CPU capabilities.
2187 *
2188 * @param pVCpu The cross context virtual CPU structure.
2189 * @param pCpu Pointer to the global HM CPU struct.
2190 */
2191DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2192{
2193#ifdef HMVMX_ALWAYS_FLUSH_TLB
2194 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2195#endif
2196 PVM pVM = pVCpu->CTX_SUFF(pVM);
2197 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2198 {
2199 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2200 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2201 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2202 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2203 default:
2204 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2205 break;
2206 }
2207
2208 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2209}
2210
2211
2212/**
2213 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2214 * TLB entries from the host TLB before VM-entry.
2215 *
2216 * @returns VBox status code.
2217 * @param pVM The cross context VM structure.
2218 */
2219static int hmR0VmxSetupTaggedTlb(PVM pVM)
2220{
2221 /*
2222 * Determine optimal flush type for Nested Paging.
2223 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2224 * guest execution (see hmR3InitFinalizeR0()).
2225 */
2226 if (pVM->hm.s.fNestedPaging)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2232 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2234 else
2235 {
2236 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2237 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2238 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2239 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2240 }
2241
2242 /* Make sure the write-back cacheable memory type for EPT is supported. */
2243 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2244 {
2245 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2246 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2247 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2248 }
2249
2250 /* EPT requires a page-walk length of 4. */
2251 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2252 {
2253 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2254 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2255 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2256 }
2257 }
2258 else
2259 {
2260 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2261 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2262 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2264 }
2265 }
2266
2267 /*
2268 * Determine optimal flush type for VPID.
2269 */
2270 if (pVM->hm.s.vmx.fVpid)
2271 {
2272 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2273 {
2274 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2275 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2276 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2277 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2278 else
2279 {
2280 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2281 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2282 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2283 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2284 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2285 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2286 pVM->hm.s.vmx.fVpid = false;
2287 }
2288 }
2289 else
2290 {
2291 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2292 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2293 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2294 pVM->hm.s.vmx.fVpid = false;
2295 }
2296 }
2297
2298 /*
2299 * Setup the handler for flushing tagged-TLBs.
2300 */
2301 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2302 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2303 else if (pVM->hm.s.fNestedPaging)
2304 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2305 else if (pVM->hm.s.vmx.fVpid)
2306 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2307 else
2308 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2309 return VINF_SUCCESS;
2310}
2311
2312
2313/**
2314 * Sets up pin-based VM-execution controls in the VMCS.
2315 *
2316 * @returns VBox status code.
2317 * @param pVM The cross context VM structure.
2318 * @param pVCpu The cross context virtual CPU structure.
2319 */
2320static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2321{
2322 AssertPtr(pVM);
2323 AssertPtr(pVCpu);
2324
2325 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2326 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2327
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2329 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2330
2331 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2332 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2333
2334 /* Enable the VMX preemption timer. */
2335 if (pVM->hm.s.vmx.fUsePreemptTimer)
2336 {
2337 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2338 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2339 }
2340
2341#if 0
2342 /* Enable posted-interrupt processing. */
2343 if (pVM->hm.s.fPostedIntrs)
2344 {
2345 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2346 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2347 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2348 }
2349#endif
2350
2351 if ((val & zap) != val)
2352 {
2353 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2354 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2355 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2356 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2357 }
2358
2359 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2360 AssertRCReturn(rc, rc);
2361
2362 pVCpu->hm.s.vmx.u32PinCtls = val;
2363 return rc;
2364}
2365
2366
2367/**
2368 * Sets up processor-based VM-execution controls in the VMCS.
2369 *
2370 * @returns VBox status code.
2371 * @param pVM The cross context VM structure.
2372 * @param pVCpu The cross context virtual CPU structure.
2373 */
2374static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2375{
2376 AssertPtr(pVM);
2377 AssertPtr(pVCpu);
2378
2379 int rc = VERR_INTERNAL_ERROR_5;
2380 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2381 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2382
2383 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2384 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2385 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2386 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2387 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2388 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2389 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2390
2391 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2392 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2393 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2394 {
2395 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2396 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2397 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2398 }
2399
2400 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2401 if (!pVM->hm.s.fNestedPaging)
2402 {
2403 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2404 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2405 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2406 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2407 }
2408
2409 /* Use TPR shadowing if supported by the CPU. */
2410 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2411 {
2412 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2413 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2414 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2415 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2416 AssertRCReturn(rc, rc);
2417
2418 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2419 /* CR8 writes cause a VM-exit based on TPR threshold. */
2420 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2421 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2422 }
2423 else
2424 {
2425 /*
2426 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2427 * Set this control only for 64-bit guests.
2428 */
2429 if (pVM->hm.s.fAllow64BitGuests)
2430 {
2431 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2432 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2433 }
2434 }
2435
2436 /* Use MSR-bitmaps if supported by the CPU. */
2437 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2438 {
2439 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2440
2441 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2442 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2443 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2444 AssertRCReturn(rc, rc);
2445
2446 /*
2447 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2448 * automatically using dedicated fields in the VMCS.
2449 */
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2455
2456#if HC_ARCH_BITS == 64
2457 /*
2458 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2459 */
2460 if (pVM->hm.s.fAllow64BitGuests)
2461 {
2462 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2463 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2464 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2465 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2466 }
2467#endif
2468 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2469 }
2470
2471 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2472 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2473 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2474
2475 if ((val & zap) != val)
2476 {
2477 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2478 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2479 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2480 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2481 }
2482
2483 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2484 AssertRCReturn(rc, rc);
2485
2486 pVCpu->hm.s.vmx.u32ProcCtls = val;
2487
2488 /*
2489 * Secondary processor-based VM-execution controls.
2490 */
2491 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2492 {
2493 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2494 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2495
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2498
2499 if (pVM->hm.s.fNestedPaging)
2500 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2501 else
2502 {
2503 /*
2504 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2505 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2506 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2507 */
2508 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2510 }
2511
2512 if (pVM->hm.s.vmx.fVpid)
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2514
2515 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2516 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2517
2518#if 0
2519 if (pVM->hm.s.fVirtApicRegs)
2520 {
2521 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2523
2524 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2525 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2526 }
2527#endif
2528
2529 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2530 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2531 * done dynamically. */
2532 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2533 {
2534 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2535 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2537 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2538 AssertRCReturn(rc, rc);
2539 }
2540
2541 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2542 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2543
2544 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2545 && pVM->hm.s.vmx.cPleGapTicks
2546 && pVM->hm.s.vmx.cPleWindowTicks)
2547 {
2548 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2549
2550 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2551 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2552 AssertRCReturn(rc, rc);
2553 }
2554
2555 if ((val & zap) != val)
2556 {
2557 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2558 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2559 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2560 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2561 }
2562
2563 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2564 AssertRCReturn(rc, rc);
2565
2566 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2567 }
2568 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569 {
2570 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2571 "available\n"));
2572 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2573 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2574 }
2575
2576 return VINF_SUCCESS;
2577}
2578
2579
2580/**
2581 * Sets up miscellaneous (everything other than Pin & Processor-based
2582 * VM-execution) control fields in the VMCS.
2583 *
2584 * @returns VBox status code.
2585 * @param pVM The cross context VM structure.
2586 * @param pVCpu The cross context virtual CPU structure.
2587 */
2588static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2589{
2590 NOREF(pVM);
2591 AssertPtr(pVM);
2592 AssertPtr(pVCpu);
2593
2594 int rc = VERR_GENERAL_FAILURE;
2595
2596 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2597#if 0
2598 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2599 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2600 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2601
2602 /*
2603 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2604 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2605 * We thus use the exception bitmap to control it rather than use both.
2606 */
2607 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2608 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2609
2610 /** @todo Explore possibility of using IO-bitmaps. */
2611 /* All IO & IOIO instructions cause VM-exits. */
2612 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2613 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2614
2615 /* Initialize the MSR-bitmap area. */
2616 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2617 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2618 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2619 AssertRCReturn(rc, rc);
2620#endif
2621
2622 /* Setup MSR auto-load/store area. */
2623 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2624 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2626 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2627 AssertRCReturn(rc, rc);
2628
2629 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2630 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2632 AssertRCReturn(rc, rc);
2633
2634 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2635 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2636 AssertRCReturn(rc, rc);
2637
2638 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2639#if 0
2640 /* Setup debug controls */
2641 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2642 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2643 AssertRCReturn(rc, rc);
2644#endif
2645
2646 return rc;
2647}
2648
2649
2650/**
2651 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2652 *
2653 * We shall setup those exception intercepts that don't change during the
2654 * lifetime of the VM here. The rest are done dynamically while loading the
2655 * guest state.
2656 *
2657 * @returns VBox status code.
2658 * @param pVM The cross context VM structure.
2659 * @param pVCpu The cross context virtual CPU structure.
2660 */
2661static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2662{
2663 AssertPtr(pVM);
2664 AssertPtr(pVCpu);
2665
2666 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2667
2668 uint32_t u32XcptBitmap = 0;
2669
2670 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2671 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2672
2673 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2674 and writes, and because recursive #DBs can cause the CPU hang, we must always
2675 intercept #DB. */
2676 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2677
2678 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2679 if (!pVM->hm.s.fNestedPaging)
2680 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2681
2682 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2683 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2684 AssertRCReturn(rc, rc);
2685 return rc;
2686}
2687
2688
2689/**
2690 * Sets up the initial guest-state mask. The guest-state mask is consulted
2691 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2692 * for the nested virtualization case (as it would cause a VM-exit).
2693 *
2694 * @param pVCpu The cross context virtual CPU structure.
2695 */
2696static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2697{
2698 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2699 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2700 return VINF_SUCCESS;
2701}
2702
2703
2704/**
2705 * Does per-VM VT-x initialization.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM The cross context VM structure.
2709 */
2710VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2711{
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714 int rc = hmR0VmxStructsAlloc(pVM);
2715 if (RT_FAILURE(rc))
2716 {
2717 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2718 return rc;
2719 }
2720
2721 return VINF_SUCCESS;
2722}
2723
2724
2725/**
2726 * Does per-VM VT-x termination.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2732{
2733 LogFlowFunc(("pVM=%p\n", pVM));
2734
2735#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2736 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2737 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2738#endif
2739 hmR0VmxStructsFree(pVM);
2740 return VINF_SUCCESS;
2741}
2742
2743
2744/**
2745 * Sets up the VM for execution under VT-x.
2746 * This function is only called once per-VM during initialization.
2747 *
2748 * @returns VBox status code.
2749 * @param pVM The cross context VM structure.
2750 */
2751VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2752{
2753 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2754 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2755
2756 LogFlowFunc(("pVM=%p\n", pVM));
2757
2758 /*
2759 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2760 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2761 */
2762 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2763 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2764 || !pVM->hm.s.vmx.pRealModeTSS))
2765 {
2766 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2767 return VERR_INTERNAL_ERROR;
2768 }
2769
2770 /* Initialize these always, see hmR3InitFinalizeR0().*/
2771 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2772 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2773
2774 /* Setup the tagged-TLB flush handlers. */
2775 int rc = hmR0VmxSetupTaggedTlb(pVM);
2776 if (RT_FAILURE(rc))
2777 {
2778 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2779 return rc;
2780 }
2781
2782 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2783 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2784#if HC_ARCH_BITS == 64
2785 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2786 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2787 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2788 {
2789 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2790 }
2791#endif
2792
2793 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2794 RTCCUINTREG uHostCR4 = ASMGetCR4();
2795 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2796 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2797
2798 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2799 {
2800 PVMCPU pVCpu = &pVM->aCpus[i];
2801 AssertPtr(pVCpu);
2802 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2803
2804 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2805 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2806
2807 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2808 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2809 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2810
2811 /* Set revision dword at the beginning of the VMCS structure. */
2812 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2813
2814 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2815 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2817 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2818
2819 /* Load this VMCS as the current VMCS. */
2820 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2826 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2827
2828 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2829 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2830 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2831
2832 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2833 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2834 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2835
2836 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2837 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2838 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2839
2840 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2841 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2842 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2843
2844#if HC_ARCH_BITS == 32
2845 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2846 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2847 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2848#endif
2849
2850 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2851 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2852 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2853 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2854
2855 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2856
2857 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2858 }
2859
2860 return VINF_SUCCESS;
2861}
2862
2863
2864/**
2865 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2866 * the VMCS.
2867 *
2868 * @returns VBox status code.
2869 * @param pVM The cross context VM structure.
2870 * @param pVCpu The cross context virtual CPU structure.
2871 */
2872DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2873{
2874 NOREF(pVM); NOREF(pVCpu);
2875
2876 RTCCUINTREG uReg = ASMGetCR0();
2877 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2878 AssertRCReturn(rc, rc);
2879
2880 uReg = ASMGetCR3();
2881 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2882 AssertRCReturn(rc, rc);
2883
2884 uReg = ASMGetCR4();
2885 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2886 AssertRCReturn(rc, rc);
2887 return rc;
2888}
2889
2890
2891#if HC_ARCH_BITS == 64
2892/**
2893 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2894 * requirements. See hmR0VmxSaveHostSegmentRegs().
2895 */
2896# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2897 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2898 { \
2899 bool fValidSelector = true; \
2900 if ((selValue) & X86_SEL_LDT) \
2901 { \
2902 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2903 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2904 } \
2905 if (fValidSelector) \
2906 { \
2907 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2908 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2909 } \
2910 (selValue) = 0; \
2911 }
2912#endif
2913
2914
2915/**
2916 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2917 * the host-state area in the VMCS.
2918 *
2919 * @returns VBox status code.
2920 * @param pVM The cross context VM structure.
2921 * @param pVCpu The cross context virtual CPU structure.
2922 */
2923DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2924{
2925 int rc = VERR_INTERNAL_ERROR_5;
2926
2927#if HC_ARCH_BITS == 64
2928 /*
2929 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2930 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2931 *
2932 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2933 * Was observed booting Solaris10u10 32-bit guest.
2934 */
2935 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2936 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2937 {
2938 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2939 pVCpu->idCpu));
2940 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2941 }
2942 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2943#else
2944 RT_NOREF(pVCpu);
2945#endif
2946
2947 /*
2948 * Host DS, ES, FS and GS segment registers.
2949 */
2950#if HC_ARCH_BITS == 64
2951 RTSEL uSelDS = ASMGetDS();
2952 RTSEL uSelES = ASMGetES();
2953 RTSEL uSelFS = ASMGetFS();
2954 RTSEL uSelGS = ASMGetGS();
2955#else
2956 RTSEL uSelDS = 0;
2957 RTSEL uSelES = 0;
2958 RTSEL uSelFS = 0;
2959 RTSEL uSelGS = 0;
2960#endif
2961
2962 /*
2963 * Host CS and SS segment registers.
2964 */
2965 RTSEL uSelCS = ASMGetCS();
2966 RTSEL uSelSS = ASMGetSS();
2967
2968 /*
2969 * Host TR segment register.
2970 */
2971 RTSEL uSelTR = ASMGetTR();
2972
2973#if HC_ARCH_BITS == 64
2974 /*
2975 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2976 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2977 */
2978 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2979 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2980 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2981 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2982# undef VMXLOCAL_ADJUST_HOST_SEG
2983#endif
2984
2985 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2986 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2987 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2988 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2989 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2990 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2991 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2992 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2993 Assert(uSelCS);
2994 Assert(uSelTR);
2995
2996 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2997#if 0
2998 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2999 Assert(uSelSS != 0);
3000#endif
3001
3002 /* Write these host selector fields into the host-state area in the VMCS. */
3003 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3004 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3005#if HC_ARCH_BITS == 64
3006 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3007 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3008 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3009 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3010#else
3011 NOREF(uSelDS);
3012 NOREF(uSelES);
3013 NOREF(uSelFS);
3014 NOREF(uSelGS);
3015#endif
3016 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3017 AssertRCReturn(rc, rc);
3018
3019 /*
3020 * Host GDTR and IDTR.
3021 */
3022 RTGDTR Gdtr;
3023 RTIDTR Idtr;
3024 RT_ZERO(Gdtr);
3025 RT_ZERO(Idtr);
3026 ASMGetGDTR(&Gdtr);
3027 ASMGetIDTR(&Idtr);
3028 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3029 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3030 AssertRCReturn(rc, rc);
3031
3032#if HC_ARCH_BITS == 64
3033 /*
3034 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3035 * maximum limit (0xffff) on every VM-exit.
3036 */
3037 if (Gdtr.cbGdt != 0xffff)
3038 {
3039 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3040 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3041 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3042 }
3043
3044 /*
3045 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3046 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3047 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3048 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3049 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3050 * hosts where we are pretty sure it won't cause trouble.
3051 */
3052# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3053 if (Idtr.cbIdt < 0x0fff)
3054# else
3055 if (Idtr.cbIdt != 0xffff)
3056# endif
3057 {
3058 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3059 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3060 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3061 }
3062#endif
3063
3064 /*
3065 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3066 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3067 */
3068 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3069 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3070 VERR_VMX_INVALID_HOST_STATE);
3071
3072 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3073#if HC_ARCH_BITS == 64
3074 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3075
3076 /*
3077 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3078 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3079 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3080 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3081 *
3082 * [1] See Intel spec. 3.5 "System Descriptor Types".
3083 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3084 */
3085 Assert(pDesc->System.u4Type == 11);
3086 if ( pDesc->System.u16LimitLow != 0x67
3087 || pDesc->System.u4LimitHigh)
3088 {
3089 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3090 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3091 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3092 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3093 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3094
3095 /* Store the GDTR here as we need it while restoring TR. */
3096 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3097 }
3098#else
3099 NOREF(pVM);
3100 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3101#endif
3102 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3103 AssertRCReturn(rc, rc);
3104
3105 /*
3106 * Host FS base and GS base.
3107 */
3108#if HC_ARCH_BITS == 64
3109 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3110 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3111 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3112 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3113 AssertRCReturn(rc, rc);
3114
3115 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3116 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3117 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3118 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3119 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3120#endif
3121 return rc;
3122}
3123
3124
3125/**
3126 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3127 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3128 * the host after every successful VM-exit.
3129 *
3130 * @returns VBox status code.
3131 * @param pVM The cross context VM structure.
3132 * @param pVCpu The cross context virtual CPU structure.
3133 *
3134 * @remarks No-long-jump zone!!!
3135 */
3136DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3137{
3138 NOREF(pVM);
3139
3140 AssertPtr(pVCpu);
3141 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3142
3143 /*
3144 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3145 * rather than swapping them on every VM-entry.
3146 */
3147 hmR0VmxLazySaveHostMsrs(pVCpu);
3148
3149 /*
3150 * Host Sysenter MSRs.
3151 */
3152 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3153#if HC_ARCH_BITS == 32
3154 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3155 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3156#else
3157 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3158 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3159#endif
3160 AssertRCReturn(rc, rc);
3161
3162 /*
3163 * Host EFER MSR.
3164 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3165 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3166 */
3167 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3168 {
3169 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3170 AssertRCReturn(rc, rc);
3171 }
3172
3173 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3174 * hmR0VmxLoadGuestExitCtls() !! */
3175
3176 return rc;
3177}
3178
3179
3180/**
3181 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3182 *
3183 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3184 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3185 * hmR0VMxLoadGuestEntryCtls().
3186 *
3187 * @returns true if we need to load guest EFER, false otherwise.
3188 * @param pVCpu The cross context virtual CPU structure.
3189 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3190 * out-of-sync. Make sure to update the required fields
3191 * before using them.
3192 *
3193 * @remarks Requires EFER, CR4.
3194 * @remarks No-long-jump zone!!!
3195 */
3196static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3197{
3198#ifdef HMVMX_ALWAYS_SWAP_EFER
3199 return true;
3200#endif
3201
3202#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3203 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3204 if (CPUMIsGuestInLongMode(pVCpu))
3205 return false;
3206#endif
3207
3208 PVM pVM = pVCpu->CTX_SUFF(pVM);
3209 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3210 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3211
3212 /*
3213 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3214 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3215 */
3216 if ( CPUMIsGuestInLongMode(pVCpu)
3217 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3218 {
3219 return true;
3220 }
3221
3222 /*
3223 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3224 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3225 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3226 */
3227 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3228 && (pMixedCtx->cr0 & X86_CR0_PG)
3229 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3230 {
3231 /* Assert that host is PAE capable. */
3232 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3233 return true;
3234 }
3235
3236 /** @todo Check the latest Intel spec. for any other bits,
3237 * like SMEP/SMAP? */
3238 return false;
3239}
3240
3241
3242/**
3243 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3244 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3245 * controls".
3246 *
3247 * @returns VBox status code.
3248 * @param pVCpu The cross context virtual CPU structure.
3249 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3250 * out-of-sync. Make sure to update the required fields
3251 * before using them.
3252 *
3253 * @remarks Requires EFER.
3254 * @remarks No-long-jump zone!!!
3255 */
3256DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3257{
3258 int rc = VINF_SUCCESS;
3259 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3260 {
3261 PVM pVM = pVCpu->CTX_SUFF(pVM);
3262 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3263 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3264
3265 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3266 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3267
3268 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3269 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3270 {
3271 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3272 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3273 }
3274 else
3275 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3276
3277 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3278 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3279 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3280 {
3281 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3282 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3283 }
3284
3285 /*
3286 * The following should -not- be set (since we're not in SMM mode):
3287 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3288 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3289 */
3290
3291 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3292 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3293
3294 if ((val & zap) != val)
3295 {
3296 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3297 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3298 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3299 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3300 }
3301
3302 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3303 AssertRCReturn(rc, rc);
3304
3305 pVCpu->hm.s.vmx.u32EntryCtls = val;
3306 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3307 }
3308 return rc;
3309}
3310
3311
3312/**
3313 * Sets up the VM-exit controls in the VMCS.
3314 *
3315 * @returns VBox status code.
3316 * @param pVCpu The cross context virtual CPU structure.
3317 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3318 * out-of-sync. Make sure to update the required fields
3319 * before using them.
3320 *
3321 * @remarks Requires EFER.
3322 */
3323DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3324{
3325 NOREF(pMixedCtx);
3326
3327 int rc = VINF_SUCCESS;
3328 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3329 {
3330 PVM pVM = pVCpu->CTX_SUFF(pVM);
3331 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3332 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3333
3334 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3335 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3336
3337 /*
3338 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3339 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3340 */
3341#if HC_ARCH_BITS == 64
3342 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3343 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3344#else
3345 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3346 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3347 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3348 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3349 {
3350 /* The switcher returns to long mode, EFER is managed by the switcher. */
3351 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3352 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3353 }
3354 else
3355 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3356#endif
3357
3358 /* If the newer VMCS fields for managing EFER exists, use it. */
3359 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3360 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3361 {
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3363 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3364 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3365 }
3366
3367 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3368 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3369
3370 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3371 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3372 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3373
3374 if ( pVM->hm.s.vmx.fUsePreemptTimer
3375 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3376 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3377
3378 if ((val & zap) != val)
3379 {
3380 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3381 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3382 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3383 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3384 }
3385
3386 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3387 AssertRCReturn(rc, rc);
3388
3389 pVCpu->hm.s.vmx.u32ExitCtls = val;
3390 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3391 }
3392 return rc;
3393}
3394
3395
3396/**
3397 * Sets the TPR threshold in the VMCS.
3398 *
3399 * @returns VBox status code.
3400 * @param pVCpu The cross context virtual CPU structure.
3401 * @param u32TprThreshold The TPR threshold (task-priority class only).
3402 */
3403DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3404{
3405 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3406 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3407 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3408}
3409
3410
3411/**
3412 * Loads the guest APIC and related state.
3413 *
3414 * @returns VBox status code.
3415 * @param pVCpu The cross context virtual CPU structure.
3416 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3417 * out-of-sync. Make sure to update the required fields
3418 * before using them.
3419 *
3420 * @remarks No-long-jump zone!!!
3421 */
3422DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423{
3424 NOREF(pMixedCtx);
3425
3426 int rc = VINF_SUCCESS;
3427 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3428 {
3429 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3430 && APICIsEnabled(pVCpu))
3431 {
3432 /*
3433 * Setup TPR shadowing.
3434 */
3435 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3436 {
3437 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3438
3439 bool fPendingIntr = false;
3440 uint8_t u8Tpr = 0;
3441 uint8_t u8PendingIntr = 0;
3442 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3443 AssertRCReturn(rc, rc);
3444
3445 /*
3446 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3447 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3448 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3449 */
3450 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3451 uint32_t u32TprThreshold = 0;
3452 if (fPendingIntr)
3453 {
3454 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3455 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3456 const uint8_t u8TprPriority = u8Tpr >> 4;
3457 if (u8PendingPriority <= u8TprPriority)
3458 u32TprThreshold = u8PendingPriority;
3459 }
3460
3461 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3462 AssertRCReturn(rc, rc);
3463 }
3464 }
3465 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3466 }
3467
3468 return rc;
3469}
3470
3471
3472/**
3473 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3474 *
3475 * @returns Guest's interruptibility-state.
3476 * @param pVCpu The cross context virtual CPU structure.
3477 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3478 * out-of-sync. Make sure to update the required fields
3479 * before using them.
3480 *
3481 * @remarks No-long-jump zone!!!
3482 */
3483DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3484{
3485 /*
3486 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3487 */
3488 uint32_t uIntrState = 0;
3489 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3490 {
3491 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3492 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3493 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3494 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3495 {
3496 if (pMixedCtx->eflags.Bits.u1IF)
3497 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3498 else
3499 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3500 }
3501 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3502 {
3503 /*
3504 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3505 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3506 */
3507 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3508 }
3509 }
3510
3511 /*
3512 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3513 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3514 * setting this would block host-NMIs and IRET will not clear the blocking.
3515 *
3516 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3517 */
3518 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3519 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3520 {
3521 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3522 }
3523
3524 return uIntrState;
3525}
3526
3527
3528/**
3529 * Loads the guest's interruptibility-state into the guest-state area in the
3530 * VMCS.
3531 *
3532 * @returns VBox status code.
3533 * @param pVCpu The cross context virtual CPU structure.
3534 * @param uIntrState The interruptibility-state to set.
3535 */
3536static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3537{
3538 NOREF(pVCpu);
3539 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3540 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3541 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3542 AssertRC(rc);
3543 return rc;
3544}
3545
3546
3547/**
3548 * Loads the exception intercepts required for guest execution in the VMCS.
3549 *
3550 * @returns VBox status code.
3551 * @param pVCpu The cross context virtual CPU structure.
3552 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3553 * out-of-sync. Make sure to update the required fields
3554 * before using them.
3555 */
3556static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3557{
3558 NOREF(pMixedCtx);
3559 int rc = VINF_SUCCESS;
3560 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3561 {
3562 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3563 if (pVCpu->hm.s.fGIMTrapXcptUD)
3564 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3565#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3566 else
3567 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3568#endif
3569
3570 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3571 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3572
3573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3577 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3578 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RIP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3605 HMCPU_CF_VALUE(pVCpu)));
3606 }
3607 return rc;
3608}
3609
3610
3611/**
3612 * Loads the guest's RSP into the guest-state area in the VMCS.
3613 *
3614 * @returns VBox status code.
3615 * @param pVCpu The cross context virtual CPU structure.
3616 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3617 * out-of-sync. Make sure to update the required fields
3618 * before using them.
3619 *
3620 * @remarks No-long-jump zone!!!
3621 */
3622static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3623{
3624 int rc = VINF_SUCCESS;
3625 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3626 {
3627 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3628 AssertRCReturn(rc, rc);
3629
3630 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3631 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3632 }
3633 return rc;
3634}
3635
3636
3637/**
3638 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3639 *
3640 * @returns VBox status code.
3641 * @param pVCpu The cross context virtual CPU structure.
3642 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3643 * out-of-sync. Make sure to update the required fields
3644 * before using them.
3645 *
3646 * @remarks No-long-jump zone!!!
3647 */
3648static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3649{
3650 int rc = VINF_SUCCESS;
3651 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3652 {
3653 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3654 Let us assert it as such and use 32-bit VMWRITE. */
3655 Assert(!(pMixedCtx->rflags.u64 >> 32));
3656 X86EFLAGS Eflags = pMixedCtx->eflags;
3657 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3658 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3659 * These will never be cleared/set, unless some other part of the VMM
3660 * code is buggy - in which case we're better of finding and fixing
3661 * those bugs than hiding them. */
3662 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3663 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3664 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3665 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3666
3667 /*
3668 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3669 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3670 */
3671 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3672 {
3673 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3674 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3675 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3676 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3677 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3678 }
3679
3680 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3681 AssertRCReturn(rc, rc);
3682
3683 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3684 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3685 }
3686 return rc;
3687}
3688
3689
3690/**
3691 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3692 *
3693 * @returns VBox status code.
3694 * @param pVCpu The cross context virtual CPU structure.
3695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3696 * out-of-sync. Make sure to update the required fields
3697 * before using them.
3698 *
3699 * @remarks No-long-jump zone!!!
3700 */
3701DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3702{
3703 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3704 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3705 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3706 AssertRCReturn(rc, rc);
3707 return rc;
3708}
3709
3710
3711/**
3712 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3713 * CR0 is partially shared with the host and we have to consider the FPU bits.
3714 *
3715 * @returns VBox status code.
3716 * @param pVCpu The cross context virtual CPU structure.
3717 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3718 * out-of-sync. Make sure to update the required fields
3719 * before using them.
3720 *
3721 * @remarks No-long-jump zone!!!
3722 */
3723static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3724{
3725 /*
3726 * Guest CR0.
3727 * Guest FPU.
3728 */
3729 int rc = VINF_SUCCESS;
3730 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3731 {
3732 Assert(!(pMixedCtx->cr0 >> 32));
3733 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3734 PVM pVM = pVCpu->CTX_SUFF(pVM);
3735
3736 /* The guest's view (read access) of its CR0 is unblemished. */
3737 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3738 AssertRCReturn(rc, rc);
3739 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3740
3741 /* Setup VT-x's view of the guest CR0. */
3742 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3743 if (pVM->hm.s.fNestedPaging)
3744 {
3745 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3746 {
3747 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3748 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3749 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3750 }
3751 else
3752 {
3753 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3754 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3755 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3756 }
3757
3758 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3760 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3761
3762 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3763 AssertRCReturn(rc, rc);
3764 }
3765 else
3766 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3767
3768 /*
3769 * Guest FPU bits.
3770 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3771 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3772 */
3773 u32GuestCR0 |= X86_CR0_NE;
3774 bool fInterceptNM = false;
3775 if (CPUMIsGuestFPUStateActive(pVCpu))
3776 {
3777 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3778 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3779 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3780 }
3781 else
3782 {
3783 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3784 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3785 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3786 }
3787
3788 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3789 bool fInterceptMF = false;
3790 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3791 fInterceptMF = true;
3792
3793 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3794 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3795 {
3796 Assert(PDMVmmDevHeapIsEnabled(pVM));
3797 Assert(pVM->hm.s.vmx.pRealModeTSS);
3798 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3799 fInterceptNM = true;
3800 fInterceptMF = true;
3801 }
3802 else
3803 {
3804 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3805 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3806 }
3807 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3808
3809 if (fInterceptNM)
3810 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3811 else
3812 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3813
3814 if (fInterceptMF)
3815 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3816 else
3817 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3818
3819 /* Additional intercepts for debugging, define these yourself explicitly. */
3820#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3821 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3822 | RT_BIT(X86_XCPT_BP)
3823 | RT_BIT(X86_XCPT_DE)
3824 | RT_BIT(X86_XCPT_NM)
3825 | RT_BIT(X86_XCPT_TS)
3826 | RT_BIT(X86_XCPT_UD)
3827 | RT_BIT(X86_XCPT_NP)
3828 | RT_BIT(X86_XCPT_SS)
3829 | RT_BIT(X86_XCPT_GP)
3830 | RT_BIT(X86_XCPT_PF)
3831 | RT_BIT(X86_XCPT_MF)
3832 ;
3833#elif defined(HMVMX_ALWAYS_TRAP_PF)
3834 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3835#endif
3836
3837 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3838
3839 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3840 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3841 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3842 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3843 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3844 else
3845 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3846
3847 u32GuestCR0 |= uSetCR0;
3848 u32GuestCR0 &= uZapCR0;
3849 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3850
3851 /* Write VT-x's view of the guest CR0 into the VMCS. */
3852 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3853 AssertRCReturn(rc, rc);
3854 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3855 uZapCR0));
3856
3857 /*
3858 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3859 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3860 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3861 */
3862 uint32_t u32CR0Mask = 0;
3863 u32CR0Mask = X86_CR0_PE
3864 | X86_CR0_NE
3865 | X86_CR0_WP
3866 | X86_CR0_PG
3867 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3868 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3869 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3870
3871 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3872 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3873 * and @bugref{6944}. */
3874#if 0
3875 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3876 u32CR0Mask &= ~X86_CR0_PE;
3877#endif
3878 if (pVM->hm.s.fNestedPaging)
3879 u32CR0Mask &= ~X86_CR0_WP;
3880
3881 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3882 if (fInterceptNM)
3883 {
3884 u32CR0Mask |= X86_CR0_TS
3885 | X86_CR0_MP;
3886 }
3887
3888 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3889 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3890 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3891 AssertRCReturn(rc, rc);
3892 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3893
3894 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3895 }
3896 return rc;
3897}
3898
3899
3900/**
3901 * Loads the guest control registers (CR3, CR4) into the guest-state area
3902 * in the VMCS.
3903 *
3904 * @returns VBox strict status code.
3905 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3906 * without unrestricted guest access and the VMMDev is not presently
3907 * mapped (e.g. EFI32).
3908 *
3909 * @param pVCpu The cross context virtual CPU structure.
3910 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3911 * out-of-sync. Make sure to update the required fields
3912 * before using them.
3913 *
3914 * @remarks No-long-jump zone!!!
3915 */
3916static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3917{
3918 int rc = VINF_SUCCESS;
3919 PVM pVM = pVCpu->CTX_SUFF(pVM);
3920
3921 /*
3922 * Guest CR2.
3923 * It's always loaded in the assembler code. Nothing to do here.
3924 */
3925
3926 /*
3927 * Guest CR3.
3928 */
3929 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3930 {
3931 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3932 if (pVM->hm.s.fNestedPaging)
3933 {
3934 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3935
3936 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3937 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3938 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3939 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3940
3941 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3942 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3943 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3944
3945 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3946 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3947 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3948 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3949 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3950 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3951 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3952
3953 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3954 AssertRCReturn(rc, rc);
3955 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3956
3957 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3958 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3959 {
3960 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3961 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3962 {
3963 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3964 AssertRCReturn(rc, rc);
3965 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3966 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3967 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3968 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3969 AssertRCReturn(rc, rc);
3970 }
3971
3972 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3973 have Unrestricted Execution to handle the guest when it's not using paging. */
3974 GCPhysGuestCR3 = pMixedCtx->cr3;
3975 }
3976 else
3977 {
3978 /*
3979 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3980 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3981 * EPT takes care of translating it to host-physical addresses.
3982 */
3983 RTGCPHYS GCPhys;
3984 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3985
3986 /* We obtain it here every time as the guest could have relocated this PCI region. */
3987 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3988 if (RT_SUCCESS(rc))
3989 { /* likely */ }
3990 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3991 {
3992 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3993 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3994 }
3995 else
3996 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3997
3998 GCPhysGuestCR3 = GCPhys;
3999 }
4000
4001 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4002 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4003 }
4004 else
4005 {
4006 /* Non-nested paging case, just use the hypervisor's CR3. */
4007 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4008
4009 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4010 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4011 }
4012 AssertRCReturn(rc, rc);
4013
4014 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4015 }
4016
4017 /*
4018 * Guest CR4.
4019 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4020 */
4021 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4022 {
4023 Assert(!(pMixedCtx->cr4 >> 32));
4024 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4025
4026 /* The guest's view of its CR4 is unblemished. */
4027 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4028 AssertRCReturn(rc, rc);
4029 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4030
4031 /* Setup VT-x's view of the guest CR4. */
4032 /*
4033 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4034 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4035 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4036 */
4037 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4038 {
4039 Assert(pVM->hm.s.vmx.pRealModeTSS);
4040 Assert(PDMVmmDevHeapIsEnabled(pVM));
4041 u32GuestCR4 &= ~X86_CR4_VME;
4042 }
4043
4044 if (pVM->hm.s.fNestedPaging)
4045 {
4046 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4047 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4048 {
4049 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4050 u32GuestCR4 |= X86_CR4_PSE;
4051 /* Our identity mapping is a 32-bit page directory. */
4052 u32GuestCR4 &= ~X86_CR4_PAE;
4053 }
4054 /* else use guest CR4.*/
4055 }
4056 else
4057 {
4058 /*
4059 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4060 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4061 */
4062 switch (pVCpu->hm.s.enmShadowMode)
4063 {
4064 case PGMMODE_REAL: /* Real-mode. */
4065 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4066 case PGMMODE_32_BIT: /* 32-bit paging. */
4067 {
4068 u32GuestCR4 &= ~X86_CR4_PAE;
4069 break;
4070 }
4071
4072 case PGMMODE_PAE: /* PAE paging. */
4073 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4074 {
4075 u32GuestCR4 |= X86_CR4_PAE;
4076 break;
4077 }
4078
4079 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4080 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4081#ifdef VBOX_ENABLE_64_BITS_GUESTS
4082 break;
4083#endif
4084 default:
4085 AssertFailed();
4086 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4087 }
4088 }
4089
4090 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4091 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4092 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4093 u32GuestCR4 |= uSetCR4;
4094 u32GuestCR4 &= uZapCR4;
4095
4096 /* Write VT-x's view of the guest CR4 into the VMCS. */
4097 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4098 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4099 AssertRCReturn(rc, rc);
4100
4101 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4102 uint32_t u32CR4Mask = X86_CR4_VME
4103 | X86_CR4_PAE
4104 | X86_CR4_PGE
4105 | X86_CR4_PSE
4106 | X86_CR4_VMXE;
4107 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4108 u32CR4Mask |= X86_CR4_OSXSAVE;
4109 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4110 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4111 AssertRCReturn(rc, rc);
4112
4113 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4114 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4115
4116 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4117 }
4118 return rc;
4119}
4120
4121
4122/**
4123 * Loads the guest debug registers into the guest-state area in the VMCS.
4124 *
4125 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4126 *
4127 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4128 *
4129 * @returns VBox status code.
4130 * @param pVCpu The cross context virtual CPU structure.
4131 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4132 * out-of-sync. Make sure to update the required fields
4133 * before using them.
4134 *
4135 * @remarks No-long-jump zone!!!
4136 */
4137static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4138{
4139 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4140 return VINF_SUCCESS;
4141
4142#ifdef VBOX_STRICT
4143 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4144 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4145 {
4146 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4147 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4148 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4149 }
4150#endif
4151
4152 int rc;
4153 PVM pVM = pVCpu->CTX_SUFF(pVM);
4154 bool fSteppingDB = false;
4155 bool fInterceptMovDRx = false;
4156 if (pVCpu->hm.s.fSingleInstruction)
4157 {
4158 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4159 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4160 {
4161 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4162 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4163 AssertRCReturn(rc, rc);
4164 Assert(fSteppingDB == false);
4165 }
4166 else
4167 {
4168 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4169 pVCpu->hm.s.fClearTrapFlag = true;
4170 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4171 fSteppingDB = true;
4172 }
4173 }
4174
4175 if ( fSteppingDB
4176 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4177 {
4178 /*
4179 * Use the combined guest and host DRx values found in the hypervisor
4180 * register set because the debugger has breakpoints active or someone
4181 * is single stepping on the host side without a monitor trap flag.
4182 *
4183 * Note! DBGF expects a clean DR6 state before executing guest code.
4184 */
4185#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4186 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4187 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4188 {
4189 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4190 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4191 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4192 }
4193 else
4194#endif
4195 if (!CPUMIsHyperDebugStateActive(pVCpu))
4196 {
4197 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4198 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4199 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4200 }
4201
4202 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4203 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4204 AssertRCReturn(rc, rc);
4205
4206 pVCpu->hm.s.fUsingHyperDR7 = true;
4207 fInterceptMovDRx = true;
4208 }
4209 else
4210 {
4211 /*
4212 * If the guest has enabled debug registers, we need to load them prior to
4213 * executing guest code so they'll trigger at the right time.
4214 */
4215 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4216 {
4217#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4218 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4219 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4220 {
4221 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4222 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4223 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4224 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4225 }
4226 else
4227#endif
4228 if (!CPUMIsGuestDebugStateActive(pVCpu))
4229 {
4230 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4231 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4232 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4233 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4234 }
4235 Assert(!fInterceptMovDRx);
4236 }
4237 /*
4238 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4239 * must intercept #DB in order to maintain a correct DR6 guest value, and
4240 * because we need to intercept it to prevent nested #DBs from hanging the
4241 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4242 */
4243#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4244 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4245 && !CPUMIsGuestDebugStateActive(pVCpu))
4246#else
4247 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4248#endif
4249 {
4250 fInterceptMovDRx = true;
4251 }
4252
4253 /* Update guest DR7. */
4254 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4255 AssertRCReturn(rc, rc);
4256
4257 pVCpu->hm.s.fUsingHyperDR7 = false;
4258 }
4259
4260 /*
4261 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4262 */
4263 if (fInterceptMovDRx)
4264 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4265 else
4266 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4267 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4268 AssertRCReturn(rc, rc);
4269
4270 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4271 return VINF_SUCCESS;
4272}
4273
4274
4275#ifdef VBOX_STRICT
4276/**
4277 * Strict function to validate segment registers.
4278 *
4279 * @remarks ASSUMES CR0 is up to date.
4280 */
4281static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4282{
4283 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4284 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4285 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4286 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4287 && ( !CPUMIsGuestInRealModeEx(pCtx)
4288 && !CPUMIsGuestInV86ModeEx(pCtx)))
4289 {
4290 /* Protected mode checks */
4291 /* CS */
4292 Assert(pCtx->cs.Attr.n.u1Present);
4293 Assert(!(pCtx->cs.Attr.u & 0xf00));
4294 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->cs.Attr.n.u1Granularity));
4297 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4298 || (pCtx->cs.Attr.n.u1Granularity));
4299 /* CS cannot be loaded with NULL in protected mode. */
4300 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4301 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4302 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4303 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4304 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4305 else
4306 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4307 /* SS */
4308 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4309 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4310 if ( !(pCtx->cr0 & X86_CR0_PE)
4311 || pCtx->cs.Attr.n.u4Type == 3)
4312 {
4313 Assert(!pCtx->ss.Attr.n.u2Dpl);
4314 }
4315 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4318 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4319 Assert(pCtx->ss.Attr.n.u1Present);
4320 Assert(!(pCtx->ss.Attr.u & 0xf00));
4321 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->ss.Attr.n.u1Granularity));
4324 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4325 || (pCtx->ss.Attr.n.u1Granularity));
4326 }
4327 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4328 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4329 {
4330 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4331 Assert(pCtx->ds.Attr.n.u1Present);
4332 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4333 Assert(!(pCtx->ds.Attr.u & 0xf00));
4334 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4335 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4336 || !(pCtx->ds.Attr.n.u1Granularity));
4337 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4338 || (pCtx->ds.Attr.n.u1Granularity));
4339 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4340 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4341 }
4342 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4343 {
4344 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4345 Assert(pCtx->es.Attr.n.u1Present);
4346 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4347 Assert(!(pCtx->es.Attr.u & 0xf00));
4348 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4349 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4350 || !(pCtx->es.Attr.n.u1Granularity));
4351 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4352 || (pCtx->es.Attr.n.u1Granularity));
4353 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4354 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4355 }
4356 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4357 {
4358 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4359 Assert(pCtx->fs.Attr.n.u1Present);
4360 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4361 Assert(!(pCtx->fs.Attr.u & 0xf00));
4362 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4363 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4364 || !(pCtx->fs.Attr.n.u1Granularity));
4365 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4366 || (pCtx->fs.Attr.n.u1Granularity));
4367 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4368 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4369 }
4370 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4371 {
4372 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4373 Assert(pCtx->gs.Attr.n.u1Present);
4374 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4375 Assert(!(pCtx->gs.Attr.u & 0xf00));
4376 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4377 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4378 || !(pCtx->gs.Attr.n.u1Granularity));
4379 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4380 || (pCtx->gs.Attr.n.u1Granularity));
4381 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4382 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4383 }
4384 /* 64-bit capable CPUs. */
4385# if HC_ARCH_BITS == 64
4386 Assert(!(pCtx->cs.u64Base >> 32));
4387 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4388 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4389 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4390# endif
4391 }
4392 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4393 || ( CPUMIsGuestInRealModeEx(pCtx)
4394 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4395 {
4396 /* Real and v86 mode checks. */
4397 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4398 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4399 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4400 {
4401 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4402 }
4403 else
4404 {
4405 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4406 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4407 }
4408
4409 /* CS */
4410 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4411 Assert(pCtx->cs.u32Limit == 0xffff);
4412 Assert(u32CSAttr == 0xf3);
4413 /* SS */
4414 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4415 Assert(pCtx->ss.u32Limit == 0xffff);
4416 Assert(u32SSAttr == 0xf3);
4417 /* DS */
4418 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4419 Assert(pCtx->ds.u32Limit == 0xffff);
4420 Assert(u32DSAttr == 0xf3);
4421 /* ES */
4422 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4423 Assert(pCtx->es.u32Limit == 0xffff);
4424 Assert(u32ESAttr == 0xf3);
4425 /* FS */
4426 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4427 Assert(pCtx->fs.u32Limit == 0xffff);
4428 Assert(u32FSAttr == 0xf3);
4429 /* GS */
4430 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4431 Assert(pCtx->gs.u32Limit == 0xffff);
4432 Assert(u32GSAttr == 0xf3);
4433 /* 64-bit capable CPUs. */
4434# if HC_ARCH_BITS == 64
4435 Assert(!(pCtx->cs.u64Base >> 32));
4436 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4437 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4438 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4439# endif
4440 }
4441}
4442#endif /* VBOX_STRICT */
4443
4444
4445/**
4446 * Writes a guest segment register into the guest-state area in the VMCS.
4447 *
4448 * @returns VBox status code.
4449 * @param pVCpu The cross context virtual CPU structure.
4450 * @param idxSel Index of the selector in the VMCS.
4451 * @param idxLimit Index of the segment limit in the VMCS.
4452 * @param idxBase Index of the segment base in the VMCS.
4453 * @param idxAccess Index of the access rights of the segment in the VMCS.
4454 * @param pSelReg Pointer to the segment selector.
4455 *
4456 * @remarks No-long-jump zone!!!
4457 */
4458static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4459 uint32_t idxAccess, PCPUMSELREG pSelReg)
4460{
4461 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4462 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4463 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4464 AssertRCReturn(rc, rc);
4465
4466 uint32_t u32Access = pSelReg->Attr.u;
4467 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4468 {
4469 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4470 u32Access = 0xf3;
4471 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4472 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4473 }
4474 else
4475 {
4476 /*
4477 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4478 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4479 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4480 * loaded in protected-mode have their attribute as 0.
4481 */
4482 if (!u32Access)
4483 u32Access = X86DESCATTR_UNUSABLE;
4484 }
4485
4486 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4487 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4488 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4489
4490 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4491 AssertRCReturn(rc, rc);
4492 return rc;
4493}
4494
4495
4496/**
4497 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4498 * into the guest-state area in the VMCS.
4499 *
4500 * @returns VBox status code.
4501 * @param pVCpu The cross context virtual CPU structure.
4502 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4503 * out-of-sync. Make sure to update the required fields
4504 * before using them.
4505 *
4506 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4507 * @remarks No-long-jump zone!!!
4508 */
4509static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4510{
4511 int rc = VERR_INTERNAL_ERROR_5;
4512 PVM pVM = pVCpu->CTX_SUFF(pVM);
4513
4514 /*
4515 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4516 */
4517 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4518 {
4519 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4520 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4521 {
4522 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4523 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4524 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4525 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4526 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4527 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4528 }
4529
4530#ifdef VBOX_WITH_REM
4531 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4532 {
4533 Assert(pVM->hm.s.vmx.pRealModeTSS);
4534 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4535 if ( pVCpu->hm.s.vmx.fWasInRealMode
4536 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4537 {
4538 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4539 in real-mode (e.g. OpenBSD 4.0) */
4540 REMFlushTBs(pVM);
4541 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4542 pVCpu->hm.s.vmx.fWasInRealMode = false;
4543 }
4544 }
4545#endif
4546 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4547 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4548 AssertRCReturn(rc, rc);
4549 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4550 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4551 AssertRCReturn(rc, rc);
4552 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4553 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4554 AssertRCReturn(rc, rc);
4555 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4556 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4557 AssertRCReturn(rc, rc);
4558 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4559 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4560 AssertRCReturn(rc, rc);
4561 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4562 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4563 AssertRCReturn(rc, rc);
4564
4565#ifdef VBOX_STRICT
4566 /* Validate. */
4567 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4568#endif
4569
4570 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4571 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4572 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4573 }
4574
4575 /*
4576 * Guest TR.
4577 */
4578 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4579 {
4580 /*
4581 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4582 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4583 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4584 */
4585 uint16_t u16Sel = 0;
4586 uint32_t u32Limit = 0;
4587 uint64_t u64Base = 0;
4588 uint32_t u32AccessRights = 0;
4589
4590 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4591 {
4592 u16Sel = pMixedCtx->tr.Sel;
4593 u32Limit = pMixedCtx->tr.u32Limit;
4594 u64Base = pMixedCtx->tr.u64Base;
4595 u32AccessRights = pMixedCtx->tr.Attr.u;
4596 }
4597 else
4598 {
4599 Assert(pVM->hm.s.vmx.pRealModeTSS);
4600 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4601
4602 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4603 RTGCPHYS GCPhys;
4604 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4605 AssertRCReturn(rc, rc);
4606
4607 X86DESCATTR DescAttr;
4608 DescAttr.u = 0;
4609 DescAttr.n.u1Present = 1;
4610 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4611
4612 u16Sel = 0;
4613 u32Limit = HM_VTX_TSS_SIZE;
4614 u64Base = GCPhys; /* in real-mode phys = virt. */
4615 u32AccessRights = DescAttr.u;
4616 }
4617
4618 /* Validate. */
4619 Assert(!(u16Sel & RT_BIT(2)));
4620 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4621 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4622 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4623 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4624 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4625 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4626 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4627 Assert( (u32Limit & 0xfff) == 0xfff
4628 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4629 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4630 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4631
4632 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4633 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4634 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4635 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4636 AssertRCReturn(rc, rc);
4637
4638 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4639 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4640 }
4641
4642 /*
4643 * Guest GDTR.
4644 */
4645 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4646 {
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4648 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4649 AssertRCReturn(rc, rc);
4650
4651 /* Validate. */
4652 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4653
4654 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4655 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4656 }
4657
4658 /*
4659 * Guest LDTR.
4660 */
4661 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4662 {
4663 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4664 uint32_t u32Access = 0;
4665 if (!pMixedCtx->ldtr.Attr.u)
4666 u32Access = X86DESCATTR_UNUSABLE;
4667 else
4668 u32Access = pMixedCtx->ldtr.Attr.u;
4669
4670 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4671 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4672 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4673 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4674 AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 if (!(u32Access & X86DESCATTR_UNUSABLE))
4678 {
4679 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4680 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4681 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4682 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4683 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4684 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4685 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4686 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4687 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4688 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4689 }
4690
4691 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4692 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4693 }
4694
4695 /*
4696 * Guest IDTR.
4697 */
4698 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4699 {
4700 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4701 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4702 AssertRCReturn(rc, rc);
4703
4704 /* Validate. */
4705 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4706
4707 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4708 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4709 }
4710
4711 return VINF_SUCCESS;
4712}
4713
4714
4715/**
4716 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4717 * areas.
4718 *
4719 * These MSRs will automatically be loaded to the host CPU on every successful
4720 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4721 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4722 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4723 *
4724 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4725 *
4726 * @returns VBox status code.
4727 * @param pVCpu The cross context virtual CPU structure.
4728 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4729 * out-of-sync. Make sure to update the required fields
4730 * before using them.
4731 *
4732 * @remarks No-long-jump zone!!!
4733 */
4734static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735{
4736 AssertPtr(pVCpu);
4737 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4738
4739 /*
4740 * MSRs that we use the auto-load/store MSR area in the VMCS.
4741 */
4742 PVM pVM = pVCpu->CTX_SUFF(pVM);
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4744 {
4745 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4746#if HC_ARCH_BITS == 32
4747 if (pVM->hm.s.fAllow64BitGuests)
4748 {
4749 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4750 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4751 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4752 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4753 AssertRCReturn(rc, rc);
4754# ifdef LOG_ENABLED
4755 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4756 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4757 {
4758 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4759 pMsr->u64Value));
4760 }
4761# endif
4762 }
4763#endif
4764 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4765 }
4766
4767 /*
4768 * Guest Sysenter MSRs.
4769 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4770 * VM-exits on WRMSRs for these MSRs.
4771 */
4772 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4773 {
4774 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4775 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4776 }
4777
4778 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4779 {
4780 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4781 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4782 }
4783
4784 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4785 {
4786 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4787 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4788 }
4789
4790 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4791 {
4792 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4793 {
4794 /*
4795 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4796 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4797 */
4798 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4799 {
4800 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4801 AssertRCReturn(rc,rc);
4802 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4803 }
4804 else
4805 {
4806 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4807 NULL /* pfAddedAndUpdated */);
4808 AssertRCReturn(rc, rc);
4809
4810 /* We need to intercept reads too, see @bugref{7386#c16}. */
4811 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4812 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4813 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4814 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4815 }
4816 }
4817 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4818 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4819 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4820 }
4821
4822 return VINF_SUCCESS;
4823}
4824
4825
4826/**
4827 * Loads the guest activity state into the guest-state area in the VMCS.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu The cross context virtual CPU structure.
4831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832 * out-of-sync. Make sure to update the required fields
4833 * before using them.
4834 *
4835 * @remarks No-long-jump zone!!!
4836 */
4837static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838{
4839 NOREF(pMixedCtx);
4840 /** @todo See if we can make use of other states, e.g.
4841 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4842 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4843 {
4844 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4845 AssertRCReturn(rc, rc);
4846
4847 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4848 }
4849 return VINF_SUCCESS;
4850}
4851
4852
4853#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4854/**
4855 * Check if guest state allows safe use of 32-bit switcher again.
4856 *
4857 * Segment bases and protected mode structures must be 32-bit addressable
4858 * because the 32-bit switcher will ignore high dword when writing these VMCS
4859 * fields. See @bugref{8432} for details.
4860 *
4861 * @returns true if safe, false if must continue to use the 64-bit switcher.
4862 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4863 * out-of-sync. Make sure to update the required fields
4864 * before using them.
4865 *
4866 * @remarks No-long-jump zone!!!
4867 */
4868static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4869{
4870 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4871 return false;
4872 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4873 return false;
4874 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4875 return false;
4876 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4877 return false;
4878 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4879 return false;
4880 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4881 return false;
4882 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4883 return false;
4884 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4885 return false;
4886 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4887 return false;
4888 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4889 return false;
4890 /* All good, bases are 32-bit. */
4891 return true;
4892}
4893#endif
4894
4895
4896/**
4897 * Sets up the appropriate function to run guest code.
4898 *
4899 * @returns VBox status code.
4900 * @param pVCpu The cross context virtual CPU structure.
4901 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4902 * out-of-sync. Make sure to update the required fields
4903 * before using them.
4904 *
4905 * @remarks No-long-jump zone!!!
4906 */
4907static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4908{
4909 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4910 {
4911#ifndef VBOX_ENABLE_64_BITS_GUESTS
4912 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4913#endif
4914 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4915#if HC_ARCH_BITS == 32
4916 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4917 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4918 {
4919 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4920 {
4921 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4922 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4923 | HM_CHANGED_VMX_ENTRY_CTLS
4924 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4925 }
4926 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4927
4928 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4929 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4930 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4931 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4932 }
4933#else
4934 /* 64-bit host. */
4935 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4936#endif
4937 }
4938 else
4939 {
4940 /* Guest is not in long mode, use the 32-bit handler. */
4941#if HC_ARCH_BITS == 32
4942 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4943 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4944 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4945 {
4946 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4947 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4948 | HM_CHANGED_VMX_ENTRY_CTLS
4949 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4950 }
4951# ifdef VBOX_ENABLE_64_BITS_GUESTS
4952 /*
4953 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4954 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4955 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4956 * the much faster 32-bit switcher again.
4957 */
4958 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4959 {
4960 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4961 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4962 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4963 }
4964 else
4965 {
4966 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4967 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4968 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4969 {
4970 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4971 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4973 | HM_CHANGED_VMX_ENTRY_CTLS
4974 | HM_CHANGED_VMX_EXIT_CTLS
4975 | HM_CHANGED_HOST_CONTEXT);
4976 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4977 }
4978 }
4979# else
4980 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4981# endif
4982#else
4983 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4984#endif
4985 }
4986 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4987 return VINF_SUCCESS;
4988}
4989
4990
4991/**
4992 * Wrapper for running the guest code in VT-x.
4993 *
4994 * @returns VBox status code, no informational status codes.
4995 * @param pVM The cross context VM structure.
4996 * @param pVCpu The cross context virtual CPU structure.
4997 * @param pCtx Pointer to the guest-CPU context.
4998 *
4999 * @remarks No-long-jump zone!!!
5000 */
5001DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5002{
5003 /*
5004 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5005 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5006 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5007 */
5008 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5009 /** @todo Add stats for resume vs launch. */
5010#ifdef VBOX_WITH_KERNEL_USING_XMM
5011 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5012#else
5013 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5014#endif
5015 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5016 return rc;
5017}
5018
5019
5020/**
5021 * Reports world-switch error and dumps some useful debug info.
5022 *
5023 * @param pVM The cross context VM structure.
5024 * @param pVCpu The cross context virtual CPU structure.
5025 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5026 * @param pCtx Pointer to the guest-CPU context.
5027 * @param pVmxTransient Pointer to the VMX transient structure (only
5028 * exitReason updated).
5029 */
5030static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5031{
5032 Assert(pVM);
5033 Assert(pVCpu);
5034 Assert(pCtx);
5035 Assert(pVmxTransient);
5036 HMVMX_ASSERT_PREEMPT_SAFE();
5037
5038 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5039 switch (rcVMRun)
5040 {
5041 case VERR_VMX_INVALID_VMXON_PTR:
5042 AssertFailed();
5043 break;
5044 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5045 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5046 {
5047 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5048 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5049 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5050 AssertRC(rc);
5051
5052 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5053 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5054 Cannot do it here as we may have been long preempted. */
5055
5056#ifdef VBOX_STRICT
5057 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5058 pVmxTransient->uExitReason));
5059 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5060 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5061 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5062 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5063 else
5064 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5065 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5066 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5067
5068 /* VMX control bits. */
5069 uint32_t u32Val;
5070 uint64_t u64Val;
5071 RTHCUINTREG uHCReg;
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5076 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5077 {
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5080 }
5081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5085 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5086 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5087 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5088 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5089 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5090 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5091 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5092 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5093 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5094 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5095 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5096 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5097 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5098 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5099 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5100 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5101 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5102 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5103 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5104 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5105 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5106 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5108 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5110 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5111 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5112 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5113 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5114 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5115 if (pVM->hm.s.fNestedPaging)
5116 {
5117 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5118 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5119 }
5120
5121 /* Guest bits. */
5122 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5123 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5124 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5125 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5126 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5127 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5128 if (pVM->hm.s.vmx.fVpid)
5129 {
5130 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5131 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5132 }
5133
5134 /* Host bits. */
5135 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5136 Log4(("Host CR0 %#RHr\n", uHCReg));
5137 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5138 Log4(("Host CR3 %#RHr\n", uHCReg));
5139 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5140 Log4(("Host CR4 %#RHr\n", uHCReg));
5141
5142 RTGDTR HostGdtr;
5143 PCX86DESCHC pDesc;
5144 ASMGetGDTR(&HostGdtr);
5145 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5146 Log4(("Host CS %#08x\n", u32Val));
5147 if (u32Val < HostGdtr.cbGdt)
5148 {
5149 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5151 }
5152
5153 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5154 Log4(("Host DS %#08x\n", u32Val));
5155 if (u32Val < HostGdtr.cbGdt)
5156 {
5157 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5158 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5159 }
5160
5161 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5162 Log4(("Host ES %#08x\n", u32Val));
5163 if (u32Val < HostGdtr.cbGdt)
5164 {
5165 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5166 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5167 }
5168
5169 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5170 Log4(("Host FS %#08x\n", u32Val));
5171 if (u32Val < HostGdtr.cbGdt)
5172 {
5173 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5174 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5175 }
5176
5177 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5178 Log4(("Host GS %#08x\n", u32Val));
5179 if (u32Val < HostGdtr.cbGdt)
5180 {
5181 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5182 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5183 }
5184
5185 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5186 Log4(("Host SS %#08x\n", u32Val));
5187 if (u32Val < HostGdtr.cbGdt)
5188 {
5189 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5190 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5191 }
5192
5193 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5194 Log4(("Host TR %#08x\n", u32Val));
5195 if (u32Val < HostGdtr.cbGdt)
5196 {
5197 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5198 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5199 }
5200
5201 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5202 Log4(("Host TR Base %#RHv\n", uHCReg));
5203 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5204 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5205 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5206 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5207 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5208 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5209 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5210 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5211 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5212 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5213 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5214 Log4(("Host RSP %#RHv\n", uHCReg));
5215 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5216 Log4(("Host RIP %#RHv\n", uHCReg));
5217# if HC_ARCH_BITS == 64
5218 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5219 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5220 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5221 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5222 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5223 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5224# endif
5225#endif /* VBOX_STRICT */
5226 break;
5227 }
5228
5229 default:
5230 /* Impossible */
5231 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5232 break;
5233 }
5234 NOREF(pVM); NOREF(pCtx);
5235}
5236
5237
5238#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5239#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5240# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5241#endif
5242#ifdef VBOX_STRICT
5243static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5244{
5245 switch (idxField)
5246 {
5247 case VMX_VMCS_GUEST_RIP:
5248 case VMX_VMCS_GUEST_RSP:
5249 case VMX_VMCS_GUEST_SYSENTER_EIP:
5250 case VMX_VMCS_GUEST_SYSENTER_ESP:
5251 case VMX_VMCS_GUEST_GDTR_BASE:
5252 case VMX_VMCS_GUEST_IDTR_BASE:
5253 case VMX_VMCS_GUEST_CS_BASE:
5254 case VMX_VMCS_GUEST_DS_BASE:
5255 case VMX_VMCS_GUEST_ES_BASE:
5256 case VMX_VMCS_GUEST_FS_BASE:
5257 case VMX_VMCS_GUEST_GS_BASE:
5258 case VMX_VMCS_GUEST_SS_BASE:
5259 case VMX_VMCS_GUEST_LDTR_BASE:
5260 case VMX_VMCS_GUEST_TR_BASE:
5261 case VMX_VMCS_GUEST_CR3:
5262 return true;
5263 }
5264 return false;
5265}
5266
5267static bool hmR0VmxIsValidReadField(uint32_t idxField)
5268{
5269 switch (idxField)
5270 {
5271 /* Read-only fields. */
5272 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5273 return true;
5274 }
5275 /* Remaining readable fields should also be writable. */
5276 return hmR0VmxIsValidWriteField(idxField);
5277}
5278#endif /* VBOX_STRICT */
5279
5280
5281/**
5282 * Executes the specified handler in 64-bit mode.
5283 *
5284 * @returns VBox status code (no informational status codes).
5285 * @param pVM The cross context VM structure.
5286 * @param pVCpu The cross context virtual CPU structure.
5287 * @param pCtx Pointer to the guest CPU context.
5288 * @param enmOp The operation to perform.
5289 * @param cParams Number of parameters.
5290 * @param paParam Array of 32-bit parameters.
5291 */
5292VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5293 uint32_t cParams, uint32_t *paParam)
5294{
5295 NOREF(pCtx);
5296
5297 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5298 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5299 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5300 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5301
5302#ifdef VBOX_STRICT
5303 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5304 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5305
5306 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5307 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5308#endif
5309
5310 /* Disable interrupts. */
5311 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5312
5313#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5314 RTCPUID idHostCpu = RTMpCpuId();
5315 CPUMR0SetLApic(pVCpu, idHostCpu);
5316#endif
5317
5318 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5319 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5320
5321 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5322 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5323 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5324
5325 /* Leave VMX Root Mode. */
5326 VMXDisable();
5327
5328 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5329
5330 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5331 CPUMSetHyperEIP(pVCpu, enmOp);
5332 for (int i = (int)cParams - 1; i >= 0; i--)
5333 CPUMPushHyper(pVCpu, paParam[i]);
5334
5335 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5336
5337 /* Call the switcher. */
5338 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5339 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5340
5341 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5342 /* Make sure the VMX instructions don't cause #UD faults. */
5343 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5344
5345 /* Re-enter VMX Root Mode */
5346 int rc2 = VMXEnable(HCPhysCpuPage);
5347 if (RT_FAILURE(rc2))
5348 {
5349 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5350 ASMSetFlags(fOldEFlags);
5351 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5352 return rc2;
5353 }
5354
5355 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5356 AssertRC(rc2);
5357 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5358 Assert(!(ASMGetFlags() & X86_EFL_IF));
5359 ASMSetFlags(fOldEFlags);
5360 return rc;
5361}
5362
5363
5364/**
5365 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5366 * supporting 64-bit guests.
5367 *
5368 * @returns VBox status code.
5369 * @param fResume Whether to VMLAUNCH or VMRESUME.
5370 * @param pCtx Pointer to the guest-CPU context.
5371 * @param pCache Pointer to the VMCS cache.
5372 * @param pVM The cross context VM structure.
5373 * @param pVCpu The cross context virtual CPU structure.
5374 */
5375DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5376{
5377 NOREF(fResume);
5378
5379 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5380 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5381
5382#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5383 pCache->uPos = 1;
5384 pCache->interPD = PGMGetInterPaeCR3(pVM);
5385 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5386#endif
5387
5388#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5389 pCache->TestIn.HCPhysCpuPage = 0;
5390 pCache->TestIn.HCPhysVmcs = 0;
5391 pCache->TestIn.pCache = 0;
5392 pCache->TestOut.HCPhysVmcs = 0;
5393 pCache->TestOut.pCache = 0;
5394 pCache->TestOut.pCtx = 0;
5395 pCache->TestOut.eflags = 0;
5396#else
5397 NOREF(pCache);
5398#endif
5399
5400 uint32_t aParam[10];
5401 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5402 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5403 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5404 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5405 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5406 aParam[5] = 0;
5407 aParam[6] = VM_RC_ADDR(pVM, pVM);
5408 aParam[7] = 0;
5409 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5410 aParam[9] = 0;
5411
5412#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5413 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5414 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5415#endif
5416 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5417
5418#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5419 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5420 Assert(pCtx->dr[4] == 10);
5421 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5422#endif
5423
5424#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5425 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5426 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5427 pVCpu->hm.s.vmx.HCPhysVmcs));
5428 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5429 pCache->TestOut.HCPhysVmcs));
5430 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5431 pCache->TestOut.pCache));
5432 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5433 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5434 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5435 pCache->TestOut.pCtx));
5436 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5437#endif
5438 return rc;
5439}
5440
5441
5442/**
5443 * Initialize the VMCS-Read cache.
5444 *
5445 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5446 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5447 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5448 * (those that have a 32-bit FULL & HIGH part).
5449 *
5450 * @returns VBox status code.
5451 * @param pVM The cross context VM structure.
5452 * @param pVCpu The cross context virtual CPU structure.
5453 */
5454static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5455{
5456#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5457{ \
5458 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5459 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5460 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5461 ++cReadFields; \
5462}
5463
5464 AssertPtr(pVM);
5465 AssertPtr(pVCpu);
5466 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5467 uint32_t cReadFields = 0;
5468
5469 /*
5470 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5471 * and serve to indicate exceptions to the rules.
5472 */
5473
5474 /* Guest-natural selector base fields. */
5475#if 0
5476 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5479#endif
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5492#if 0
5493 /* Unused natural width guest-state fields. */
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5496#endif
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5499
5500 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5501#if 0
5502 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5511#endif
5512
5513 /* Natural width guest-state fields. */
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5515#if 0
5516 /* Currently unused field. */
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5518#endif
5519
5520 if (pVM->hm.s.fNestedPaging)
5521 {
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5523 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5524 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5525 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5526 }
5527 else
5528 {
5529 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5530 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5531 }
5532
5533#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5534 return VINF_SUCCESS;
5535}
5536
5537
5538/**
5539 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5540 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5541 * darwin, running 64-bit guests).
5542 *
5543 * @returns VBox status code.
5544 * @param pVCpu The cross context virtual CPU structure.
5545 * @param idxField The VMCS field encoding.
5546 * @param u64Val 16, 32 or 64-bit value.
5547 */
5548VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5549{
5550 int rc;
5551 switch (idxField)
5552 {
5553 /*
5554 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5555 */
5556 /* 64-bit Control fields. */
5557 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5558 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5559 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5560 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5561 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5562 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5563 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5564 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5565 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5566 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5567 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5568 case VMX_VMCS64_CTRL_EPTP_FULL:
5569 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5570 /* 64-bit Guest-state fields. */
5571 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5572 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5573 case VMX_VMCS64_GUEST_PAT_FULL:
5574 case VMX_VMCS64_GUEST_EFER_FULL:
5575 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5576 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5577 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5578 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5579 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5580 /* 64-bit Host-state fields. */
5581 case VMX_VMCS64_HOST_PAT_FULL:
5582 case VMX_VMCS64_HOST_EFER_FULL:
5583 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5584 {
5585 rc = VMXWriteVmcs32(idxField, u64Val);
5586 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5587 break;
5588 }
5589
5590 /*
5591 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5592 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5593 */
5594 /* Natural-width Guest-state fields. */
5595 case VMX_VMCS_GUEST_CR3:
5596 case VMX_VMCS_GUEST_ES_BASE:
5597 case VMX_VMCS_GUEST_CS_BASE:
5598 case VMX_VMCS_GUEST_SS_BASE:
5599 case VMX_VMCS_GUEST_DS_BASE:
5600 case VMX_VMCS_GUEST_FS_BASE:
5601 case VMX_VMCS_GUEST_GS_BASE:
5602 case VMX_VMCS_GUEST_LDTR_BASE:
5603 case VMX_VMCS_GUEST_TR_BASE:
5604 case VMX_VMCS_GUEST_GDTR_BASE:
5605 case VMX_VMCS_GUEST_IDTR_BASE:
5606 case VMX_VMCS_GUEST_RSP:
5607 case VMX_VMCS_GUEST_RIP:
5608 case VMX_VMCS_GUEST_SYSENTER_ESP:
5609 case VMX_VMCS_GUEST_SYSENTER_EIP:
5610 {
5611 if (!(u64Val >> 32))
5612 {
5613 /* If this field is 64-bit, VT-x will zero out the top bits. */
5614 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5615 }
5616 else
5617 {
5618 /* Assert that only the 32->64 switcher case should ever come here. */
5619 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5620 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5621 }
5622 break;
5623 }
5624
5625 default:
5626 {
5627 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5628 rc = VERR_INVALID_PARAMETER;
5629 break;
5630 }
5631 }
5632 AssertRCReturn(rc, rc);
5633 return rc;
5634}
5635
5636
5637/**
5638 * Queue up a VMWRITE by using the VMCS write cache.
5639 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5640 *
5641 * @param pVCpu The cross context virtual CPU structure.
5642 * @param idxField The VMCS field encoding.
5643 * @param u64Val 16, 32 or 64-bit value.
5644 */
5645VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5646{
5647 AssertPtr(pVCpu);
5648 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5649
5650 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5651 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5652
5653 /* Make sure there are no duplicates. */
5654 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5655 {
5656 if (pCache->Write.aField[i] == idxField)
5657 {
5658 pCache->Write.aFieldVal[i] = u64Val;
5659 return VINF_SUCCESS;
5660 }
5661 }
5662
5663 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5664 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5665 pCache->Write.cValidEntries++;
5666 return VINF_SUCCESS;
5667}
5668#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5669
5670
5671/**
5672 * Sets up the usage of TSC-offsetting and updates the VMCS.
5673 *
5674 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5675 * VMX preemption timer.
5676 *
5677 * @returns VBox status code.
5678 * @param pVM The cross context VM structure.
5679 * @param pVCpu The cross context virtual CPU structure.
5680 *
5681 * @remarks No-long-jump zone!!!
5682 */
5683static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5684{
5685 int rc;
5686 bool fOffsettedTsc;
5687 bool fParavirtTsc;
5688 if (pVM->hm.s.vmx.fUsePreemptTimer)
5689 {
5690 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5691 &fOffsettedTsc, &fParavirtTsc);
5692
5693 /* Make sure the returned values have sane upper and lower boundaries. */
5694 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5695 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5696 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5697 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5698
5699 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5700 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5701 }
5702 else
5703 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5704
5705 /** @todo later optimize this to be done elsewhere and not before every
5706 * VM-entry. */
5707 if (fParavirtTsc)
5708 {
5709 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5710 information before every VM-entry, hence disable it for performance sake. */
5711#if 0
5712 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5713 AssertRC(rc);
5714#endif
5715 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5716 }
5717
5718 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5719 {
5720 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5721 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5722
5723 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5724 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5725 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5726 }
5727 else
5728 {
5729 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5730 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5731 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5732 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5733 }
5734}
5735
5736
5737#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5738/**
5739 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5740 * VM-exit interruption info type.
5741 *
5742 * @returns The IEM exception flags.
5743 * @param uVector The event vector.
5744 * @param uVmxVectorType The VMX event type.
5745 *
5746 * @remarks This function currently only constructs flags required for
5747 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5748 * and CR2 aspects of an exception are not included).
5749 */
5750static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5751{
5752 uint32_t fIemXcptFlags;
5753 switch (uVmxVectorType)
5754 {
5755 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5756 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5757 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5758 break;
5759
5760 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5761 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5762 break;
5763
5764 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5765 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5766 break;
5767
5768 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5769 {
5770 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5771 if (uVector == X86_XCPT_BP)
5772 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5773 else if (uVector == X86_XCPT_OF)
5774 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5775 else
5776 {
5777 fIemXcptFlags = 0;
5778 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5779 }
5780 break;
5781 }
5782
5783 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5784 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5785 break;
5786
5787 default:
5788 fIemXcptFlags = 0;
5789 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5790 break;
5791 }
5792 return fIemXcptFlags;
5793}
5794
5795#else
5796/**
5797 * Determines if an exception is a contributory exception.
5798 *
5799 * Contributory exceptions are ones which can cause double-faults unless the
5800 * original exception was a benign exception. Page-fault is intentionally not
5801 * included here as it's a conditional contributory exception.
5802 *
5803 * @returns true if the exception is contributory, false otherwise.
5804 * @param uVector The exception vector.
5805 */
5806DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5807{
5808 switch (uVector)
5809 {
5810 case X86_XCPT_GP:
5811 case X86_XCPT_SS:
5812 case X86_XCPT_NP:
5813 case X86_XCPT_TS:
5814 case X86_XCPT_DE:
5815 return true;
5816 default:
5817 break;
5818 }
5819 return false;
5820}
5821#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5822
5823
5824/**
5825 * Sets an event as a pending event to be injected into the guest.
5826 *
5827 * @param pVCpu The cross context virtual CPU structure.
5828 * @param u32IntInfo The VM-entry interruption-information field.
5829 * @param cbInstr The VM-entry instruction length in bytes (for software
5830 * interrupts, exceptions and privileged software
5831 * exceptions).
5832 * @param u32ErrCode The VM-entry exception error code.
5833 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5834 * page-fault.
5835 *
5836 * @remarks Statistics counter assumes this is a guest event being injected or
5837 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5838 * always incremented.
5839 */
5840DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5841 RTGCUINTPTR GCPtrFaultAddress)
5842{
5843 Assert(!pVCpu->hm.s.Event.fPending);
5844 pVCpu->hm.s.Event.fPending = true;
5845 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5846 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5847 pVCpu->hm.s.Event.cbInstr = cbInstr;
5848 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5849}
5850
5851
5852/**
5853 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5854 *
5855 * @param pVCpu The cross context virtual CPU structure.
5856 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5857 * out-of-sync. Make sure to update the required fields
5858 * before using them.
5859 */
5860DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5861{
5862 NOREF(pMixedCtx);
5863 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5864 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5865 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5866 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5867}
5868
5869
5870/**
5871 * Handle a condition that occurred while delivering an event through the guest
5872 * IDT.
5873 *
5874 * @returns Strict VBox status code (i.e. informational status codes too).
5875 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5876 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5877 * to continue execution of the guest which will delivery the \#DF.
5878 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5879 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5880 *
5881 * @param pVCpu The cross context virtual CPU structure.
5882 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5883 * out-of-sync. Make sure to update the required fields
5884 * before using them.
5885 * @param pVmxTransient Pointer to the VMX transient structure.
5886 *
5887 * @remarks No-long-jump zone!!!
5888 */
5889static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5890{
5891 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5892
5893 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5894 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5895
5896 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5897 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5898 {
5899 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5900 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5901#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5902 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5903 IEMXCPTRAISE enmRaise;
5904 IEMXCPTRAISEINFO fRaiseInfo;
5905 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5906 {
5907 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5908 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5909 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5910 AssertMsg( uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
5911 || uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI,
5912 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. type %#x!\n", uExitVectorType));
5913 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector,
5914 &fRaiseInfo);
5915
5916 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_XCPT | IEMXCPTRAISEINFO_NMI_XCPT))
5917 {
5918 /*
5919 * For some reason we go back to the interpreter if delivery of an external interrupt or
5920 * NMI causes an exception, see hmR0VmxExitXcptOrNmi. As long as we go back to the interpret
5921 * we need to keep the previous (first) event pending, hence this hack.
5922 */
5923 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5924
5925 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5926 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5927 pVmxTransient->fVectoringPF = true;
5928 }
5929 }
5930 else
5931 {
5932 /*
5933 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5934 * interruption-information will not be valid as it's not an exception and we end up here.
5935 *
5936 * If the event was an external interrupt or hardare exception (incl. NMI) it is sufficient to
5937 * reflect this event to the guest after handling the VM-exit.
5938 *
5939 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5940 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5941 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5942 * event, as that can cause premature trips to ring-3 before injection and involve TRPM which
5943 * currently has no way of storing that the exceptions were caused by these special instructions.
5944 */
5945 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5946 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5947 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT)
5948 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5949 else
5950 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5951 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5952 }
5953
5954 /*
5955 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5956 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5957 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5958 * subsequent VM-entry would fail.
5959 *
5960 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5961 */
5962 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5963 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5964 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5965 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5966 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5967 {
5968 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5969 }
5970
5971 switch (enmRaise)
5972 {
5973 case IEMXCPTRAISE_CURRENT_XCPT:
5974 case IEMXCPTRAISE_REEXEC_INSTR:
5975 Assert(rcStrict == VINF_SUCCESS);
5976 break;
5977
5978 case IEMXCPTRAISE_PREV_EVENT:
5979 {
5980 /*
5981 * Re-raise the previous (first) exception/interrupt as delivery caused a premature VM-exit.
5982 */
5983 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5984 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5985 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5986
5987 uint32_t u32ErrCode;
5988 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5989 {
5990 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5991 AssertRCReturn(rc2, rc2);
5992 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5993 }
5994 else
5995 u32ErrCode = 0;
5996
5997 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5998 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5999 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6000 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6001
6002 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6003 pVCpu->hm.s.Event.u32ErrCode));
6004 Assert(rcStrict == VINF_SUCCESS);
6005 break;
6006 }
6007
6008 case IEMXCPTRAISE_DOUBLE_FAULT:
6009 {
6010 /*
6011 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6012 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6013 */
6014 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6015 {
6016 pVmxTransient->fVectoringDoublePF = true;
6017 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6018 pMixedCtx->cr2));
6019 rcStrict = VINF_SUCCESS;
6020 }
6021 else
6022 {
6023 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6024 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6025 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6026 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6027 rcStrict = VINF_HM_DOUBLE_FAULT;
6028 }
6029 break;
6030 }
6031
6032 case IEMXCPTRAISE_TRIPLE_FAULT:
6033 {
6034 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6035 uExitVector));
6036 rcStrict = VINF_EM_RESET;
6037 break;
6038 }
6039
6040 case IEMXCPTRAISE_CPU_HANG:
6041 {
6042 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6043 rcStrict = VERR_EM_GUEST_CPU_HANG;
6044 break;
6045 }
6046
6047 default:
6048 {
6049 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6050 rcStrict = VERR_VMX_IPE_2;
6051 break;
6052 }
6053 }
6054#else
6055 typedef enum
6056 {
6057 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6058 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6059 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6060 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6061 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6062 } VMXREFLECTXCPT;
6063
6064 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6065 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6066 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6067 {
6068 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6069 {
6070 enmReflect = VMXREFLECTXCPT_XCPT;
6071#ifdef VBOX_STRICT
6072 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6073 && uExitVector == X86_XCPT_PF)
6074 {
6075 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6076 }
6077#endif
6078 if ( uExitVector == X86_XCPT_PF
6079 && uIdtVector == X86_XCPT_PF)
6080 {
6081 pVmxTransient->fVectoringDoublePF = true;
6082 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6083 }
6084 else if ( uExitVector == X86_XCPT_AC
6085 && uIdtVector == X86_XCPT_AC)
6086 {
6087 enmReflect = VMXREFLECTXCPT_HANG;
6088 Log4(("IDT: Nested #AC - Bad guest\n"));
6089 }
6090 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6091 && hmR0VmxIsContributoryXcpt(uExitVector)
6092 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6093 || uIdtVector == X86_XCPT_PF))
6094 {
6095 enmReflect = VMXREFLECTXCPT_DF;
6096 }
6097 else if (uIdtVector == X86_XCPT_DF)
6098 enmReflect = VMXREFLECTXCPT_TF;
6099 }
6100 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6101 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6102 {
6103 /*
6104 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6105 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6106 */
6107 enmReflect = VMXREFLECTXCPT_XCPT;
6108
6109 if (uExitVector == X86_XCPT_PF)
6110 {
6111 pVmxTransient->fVectoringPF = true;
6112 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6113 }
6114 }
6115 }
6116 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6117 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6118 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6119 {
6120 /*
6121 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6122 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6123 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6124 */
6125 enmReflect = VMXREFLECTXCPT_XCPT;
6126 }
6127
6128 /*
6129 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6130 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6131 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6132 *
6133 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6134 */
6135 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6136 && enmReflect == VMXREFLECTXCPT_XCPT
6137 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6138 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6139 {
6140 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6141 }
6142
6143 switch (enmReflect)
6144 {
6145 case VMXREFLECTXCPT_XCPT:
6146 {
6147 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6148 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6149 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6150
6151 uint32_t u32ErrCode = 0;
6152 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6153 {
6154 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6155 AssertRCReturn(rc2, rc2);
6156 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6157 }
6158
6159 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6160 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6161 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6162 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6163 rcStrict = VINF_SUCCESS;
6164 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6165 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6166
6167 break;
6168 }
6169
6170 case VMXREFLECTXCPT_DF:
6171 {
6172 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6173 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6174 rcStrict = VINF_HM_DOUBLE_FAULT;
6175 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6176 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6177
6178 break;
6179 }
6180
6181 case VMXREFLECTXCPT_TF:
6182 {
6183 rcStrict = VINF_EM_RESET;
6184 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6185 uExitVector));
6186 break;
6187 }
6188
6189 case VMXREFLECTXCPT_HANG:
6190 {
6191 rcStrict = VERR_EM_GUEST_CPU_HANG;
6192 break;
6193 }
6194
6195 default:
6196 Assert(rcStrict == VINF_SUCCESS);
6197 break;
6198 }
6199#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6200 }
6201 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6202 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6203 && uExitVector != X86_XCPT_DF
6204 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6205 {
6206 /*
6207 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6208 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6209 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6210 */
6211 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6212 {
6213 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6214 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6215 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6216 }
6217 }
6218
6219 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6220 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6221 return rcStrict;
6222}
6223
6224
6225/**
6226 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6227 *
6228 * @returns VBox status code.
6229 * @param pVCpu The cross context virtual CPU structure.
6230 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6231 * out-of-sync. Make sure to update the required fields
6232 * before using them.
6233 *
6234 * @remarks No-long-jump zone!!!
6235 */
6236static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6237{
6238 NOREF(pMixedCtx);
6239
6240 /*
6241 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6242 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6243 */
6244 VMMRZCallRing3Disable(pVCpu);
6245 HM_DISABLE_PREEMPT();
6246
6247 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6248 {
6249#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6250 * and 'dbgc-init' containing:
6251 * sxe "xcpt_de"
6252 * sxe "xcpt_bp"
6253 * sxi "xcpt_gp"
6254 * sxi "xcpt_ss"
6255 * sxi "xcpt_np"
6256 */
6257 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6258#endif
6259 uint32_t uVal = 0;
6260 uint32_t uShadow = 0;
6261 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6262 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6263 AssertRCReturn(rc, rc);
6264
6265 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6266 CPUMSetGuestCR0(pVCpu, uVal);
6267 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6268 }
6269
6270 HM_RESTORE_PREEMPT();
6271 VMMRZCallRing3Enable(pVCpu);
6272 return VINF_SUCCESS;
6273}
6274
6275
6276/**
6277 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6278 *
6279 * @returns VBox status code.
6280 * @param pVCpu The cross context virtual CPU structure.
6281 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6282 * out-of-sync. Make sure to update the required fields
6283 * before using them.
6284 *
6285 * @remarks No-long-jump zone!!!
6286 */
6287static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6288{
6289 NOREF(pMixedCtx);
6290
6291 int rc = VINF_SUCCESS;
6292 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6293 {
6294 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6295 uint32_t uVal = 0;
6296 uint32_t uShadow = 0;
6297 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6298 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6299 AssertRCReturn(rc, rc);
6300
6301 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6302 CPUMSetGuestCR4(pVCpu, uVal);
6303 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6304 }
6305 return rc;
6306}
6307
6308
6309/**
6310 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6311 *
6312 * @returns VBox status code.
6313 * @param pVCpu The cross context virtual CPU structure.
6314 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6315 * out-of-sync. Make sure to update the required fields
6316 * before using them.
6317 *
6318 * @remarks No-long-jump zone!!!
6319 */
6320static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6321{
6322 int rc = VINF_SUCCESS;
6323 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6324 {
6325 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6326 uint64_t u64Val = 0;
6327 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6328 AssertRCReturn(rc, rc);
6329
6330 pMixedCtx->rip = u64Val;
6331 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6332 }
6333 return rc;
6334}
6335
6336
6337/**
6338 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6339 *
6340 * @returns VBox status code.
6341 * @param pVCpu The cross context virtual CPU structure.
6342 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6343 * out-of-sync. Make sure to update the required fields
6344 * before using them.
6345 *
6346 * @remarks No-long-jump zone!!!
6347 */
6348static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6349{
6350 int rc = VINF_SUCCESS;
6351 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6352 {
6353 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6354 uint64_t u64Val = 0;
6355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6356 AssertRCReturn(rc, rc);
6357
6358 pMixedCtx->rsp = u64Val;
6359 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6360 }
6361 return rc;
6362}
6363
6364
6365/**
6366 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6367 *
6368 * @returns VBox status code.
6369 * @param pVCpu The cross context virtual CPU structure.
6370 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6371 * out-of-sync. Make sure to update the required fields
6372 * before using them.
6373 *
6374 * @remarks No-long-jump zone!!!
6375 */
6376static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6377{
6378 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6379 {
6380 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6381 uint32_t uVal = 0;
6382 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6383 AssertRCReturn(rc, rc);
6384
6385 pMixedCtx->eflags.u32 = uVal;
6386 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6387 {
6388 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6389 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6390
6391 pMixedCtx->eflags.Bits.u1VM = 0;
6392 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6393 }
6394
6395 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6396 }
6397 return VINF_SUCCESS;
6398}
6399
6400
6401/**
6402 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6403 * guest-CPU context.
6404 */
6405DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6406{
6407 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6408 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6409 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6410 return rc;
6411}
6412
6413
6414/**
6415 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6416 * from the guest-state area in the VMCS.
6417 *
6418 * @param pVCpu The cross context virtual CPU structure.
6419 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6420 * out-of-sync. Make sure to update the required fields
6421 * before using them.
6422 *
6423 * @remarks No-long-jump zone!!!
6424 */
6425static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6426{
6427 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6428 {
6429 uint32_t uIntrState = 0;
6430 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6431 AssertRC(rc);
6432
6433 if (!uIntrState)
6434 {
6435 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6436 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6437
6438 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6439 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6440 }
6441 else
6442 {
6443 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6444 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6445 {
6446 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6447 AssertRC(rc);
6448 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6449 AssertRC(rc);
6450
6451 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6452 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6453 }
6454 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6455 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6456
6457 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6458 {
6459 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6460 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6461 }
6462 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6463 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6464 }
6465
6466 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6467 }
6468}
6469
6470
6471/**
6472 * Saves the guest's activity state.
6473 *
6474 * @returns VBox status code.
6475 * @param pVCpu The cross context virtual CPU structure.
6476 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6477 * out-of-sync. Make sure to update the required fields
6478 * before using them.
6479 *
6480 * @remarks No-long-jump zone!!!
6481 */
6482static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6483{
6484 NOREF(pMixedCtx);
6485 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6486 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6487 return VINF_SUCCESS;
6488}
6489
6490
6491/**
6492 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6493 * the current VMCS into the guest-CPU context.
6494 *
6495 * @returns VBox status code.
6496 * @param pVCpu The cross context virtual CPU structure.
6497 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6498 * out-of-sync. Make sure to update the required fields
6499 * before using them.
6500 *
6501 * @remarks No-long-jump zone!!!
6502 */
6503static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6504{
6505 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6506 {
6507 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6508 uint32_t u32Val = 0;
6509 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6510 pMixedCtx->SysEnter.cs = u32Val;
6511 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6512 }
6513
6514 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6515 {
6516 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6517 uint64_t u64Val = 0;
6518 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6519 pMixedCtx->SysEnter.eip = u64Val;
6520 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6521 }
6522 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6523 {
6524 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6525 uint64_t u64Val = 0;
6526 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6527 pMixedCtx->SysEnter.esp = u64Val;
6528 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6529 }
6530 return VINF_SUCCESS;
6531}
6532
6533
6534/**
6535 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6536 * the CPU back into the guest-CPU context.
6537 *
6538 * @returns VBox status code.
6539 * @param pVCpu The cross context virtual CPU structure.
6540 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6541 * out-of-sync. Make sure to update the required fields
6542 * before using them.
6543 *
6544 * @remarks No-long-jump zone!!!
6545 */
6546static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6547{
6548 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6549 VMMRZCallRing3Disable(pVCpu);
6550 HM_DISABLE_PREEMPT();
6551
6552 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6553 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6554 {
6555 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS));
6556 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6557 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6558 }
6559
6560 HM_RESTORE_PREEMPT();
6561 VMMRZCallRing3Enable(pVCpu);
6562
6563 return VINF_SUCCESS;
6564}
6565
6566
6567/**
6568 * Saves the auto load/store'd guest MSRs from the current VMCS into
6569 * the guest-CPU context.
6570 *
6571 * @returns VBox status code.
6572 * @param pVCpu The cross context virtual CPU structure.
6573 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6574 * out-of-sync. Make sure to update the required fields
6575 * before using them.
6576 *
6577 * @remarks No-long-jump zone!!!
6578 */
6579static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6580{
6581 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6582 return VINF_SUCCESS;
6583
6584 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6585 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6586 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6587 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6588 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6589 {
6590 switch (pMsr->u32Msr)
6591 {
6592 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6593 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6594 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6595 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6596 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6597 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6598 break;
6599
6600 default:
6601 {
6602 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6603 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6604 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6605 }
6606 }
6607 }
6608
6609 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6610 return VINF_SUCCESS;
6611}
6612
6613
6614/**
6615 * Saves the guest control registers from the current VMCS into the guest-CPU
6616 * context.
6617 *
6618 * @returns VBox status code.
6619 * @param pVCpu The cross context virtual CPU structure.
6620 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6621 * out-of-sync. Make sure to update the required fields
6622 * before using them.
6623 *
6624 * @remarks No-long-jump zone!!!
6625 */
6626static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6627{
6628 /* Guest CR0. Guest FPU. */
6629 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6630 AssertRCReturn(rc, rc);
6631
6632 /* Guest CR4. */
6633 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6634 AssertRCReturn(rc, rc);
6635
6636 /* Guest CR2 - updated always during the world-switch or in #PF. */
6637 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6638 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6639 {
6640 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6641 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6642 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6643
6644 PVM pVM = pVCpu->CTX_SUFF(pVM);
6645 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6646 || ( pVM->hm.s.fNestedPaging
6647 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6648 {
6649 uint64_t u64Val = 0;
6650 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6651 if (pMixedCtx->cr3 != u64Val)
6652 {
6653 CPUMSetGuestCR3(pVCpu, u64Val);
6654 if (VMMRZCallRing3IsEnabled(pVCpu))
6655 {
6656 PGMUpdateCR3(pVCpu, u64Val);
6657 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6658 }
6659 else
6660 {
6661 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6662 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6663 }
6664 }
6665
6666 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6667 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6668 {
6669 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6670 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6671 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6672 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6673 AssertRCReturn(rc, rc);
6674
6675 if (VMMRZCallRing3IsEnabled(pVCpu))
6676 {
6677 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6678 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6679 }
6680 else
6681 {
6682 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6683 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6684 }
6685 }
6686 }
6687
6688 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6689 }
6690
6691 /*
6692 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6693 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6694 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6695 *
6696 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6697 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6698 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6699 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6700 *
6701 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6702 */
6703 if (VMMRZCallRing3IsEnabled(pVCpu))
6704 {
6705 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6706 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6707
6708 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6709 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6710
6711 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6712 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6713 }
6714
6715 return rc;
6716}
6717
6718
6719/**
6720 * Reads a guest segment register from the current VMCS into the guest-CPU
6721 * context.
6722 *
6723 * @returns VBox status code.
6724 * @param pVCpu The cross context virtual CPU structure.
6725 * @param idxSel Index of the selector in the VMCS.
6726 * @param idxLimit Index of the segment limit in the VMCS.
6727 * @param idxBase Index of the segment base in the VMCS.
6728 * @param idxAccess Index of the access rights of the segment in the VMCS.
6729 * @param pSelReg Pointer to the segment selector.
6730 *
6731 * @remarks No-long-jump zone!!!
6732 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6733 * macro as that takes care of whether to read from the VMCS cache or
6734 * not.
6735 */
6736DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6737 PCPUMSELREG pSelReg)
6738{
6739 NOREF(pVCpu);
6740
6741 uint32_t u32Val = 0;
6742 int rc = VMXReadVmcs32(idxSel, &u32Val);
6743 AssertRCReturn(rc, rc);
6744 pSelReg->Sel = (uint16_t)u32Val;
6745 pSelReg->ValidSel = (uint16_t)u32Val;
6746 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6747
6748 rc = VMXReadVmcs32(idxLimit, &u32Val);
6749 AssertRCReturn(rc, rc);
6750 pSelReg->u32Limit = u32Val;
6751
6752 uint64_t u64Val = 0;
6753 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6754 AssertRCReturn(rc, rc);
6755 pSelReg->u64Base = u64Val;
6756
6757 rc = VMXReadVmcs32(idxAccess, &u32Val);
6758 AssertRCReturn(rc, rc);
6759 pSelReg->Attr.u = u32Val;
6760
6761 /*
6762 * If VT-x marks the segment as unusable, most other bits remain undefined:
6763 * - For CS the L, D and G bits have meaning.
6764 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6765 * - For the remaining data segments no bits are defined.
6766 *
6767 * The present bit and the unusable bit has been observed to be set at the
6768 * same time (the selector was supposed to be invalid as we started executing
6769 * a V8086 interrupt in ring-0).
6770 *
6771 * What should be important for the rest of the VBox code, is that the P bit is
6772 * cleared. Some of the other VBox code recognizes the unusable bit, but
6773 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6774 * safe side here, we'll strip off P and other bits we don't care about. If
6775 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6776 *
6777 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6778 */
6779 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6780 {
6781 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6782
6783 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6784 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6785 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6786
6787 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6788#ifdef DEBUG_bird
6789 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6790 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6791 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6792#endif
6793 }
6794 return VINF_SUCCESS;
6795}
6796
6797
6798#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6799# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6800 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6801 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6802#else
6803# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6804 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6805 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6806#endif
6807
6808
6809/**
6810 * Saves the guest segment registers from the current VMCS into the guest-CPU
6811 * context.
6812 *
6813 * @returns VBox status code.
6814 * @param pVCpu The cross context virtual CPU structure.
6815 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6816 * out-of-sync. Make sure to update the required fields
6817 * before using them.
6818 *
6819 * @remarks No-long-jump zone!!!
6820 */
6821static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6822{
6823 /* Guest segment registers. */
6824 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6825 {
6826 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6827 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6828 AssertRCReturn(rc, rc);
6829
6830 rc = VMXLOCAL_READ_SEG(CS, cs);
6831 rc |= VMXLOCAL_READ_SEG(SS, ss);
6832 rc |= VMXLOCAL_READ_SEG(DS, ds);
6833 rc |= VMXLOCAL_READ_SEG(ES, es);
6834 rc |= VMXLOCAL_READ_SEG(FS, fs);
6835 rc |= VMXLOCAL_READ_SEG(GS, gs);
6836 AssertRCReturn(rc, rc);
6837
6838 /* Restore segment attributes for real-on-v86 mode hack. */
6839 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6840 {
6841 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6842 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6843 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6844 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6845 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6846 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6847 }
6848 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6849 }
6850
6851 return VINF_SUCCESS;
6852}
6853
6854
6855/**
6856 * Saves the guest descriptor table registers and task register from the current
6857 * VMCS into the guest-CPU context.
6858 *
6859 * @returns VBox status code.
6860 * @param pVCpu The cross context virtual CPU structure.
6861 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6862 * out-of-sync. Make sure to update the required fields
6863 * before using them.
6864 *
6865 * @remarks No-long-jump zone!!!
6866 */
6867static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6868{
6869 int rc = VINF_SUCCESS;
6870
6871 /* Guest LDTR. */
6872 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6873 {
6874 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6875 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6876 AssertRCReturn(rc, rc);
6877 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6878 }
6879
6880 /* Guest GDTR. */
6881 uint64_t u64Val = 0;
6882 uint32_t u32Val = 0;
6883 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6884 {
6885 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6886 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6887 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6888 pMixedCtx->gdtr.pGdt = u64Val;
6889 pMixedCtx->gdtr.cbGdt = u32Val;
6890 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6891 }
6892
6893 /* Guest IDTR. */
6894 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6895 {
6896 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6897 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6898 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6899 pMixedCtx->idtr.pIdt = u64Val;
6900 pMixedCtx->idtr.cbIdt = u32Val;
6901 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6902 }
6903
6904 /* Guest TR. */
6905 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6906 {
6907 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6908 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6909 AssertRCReturn(rc, rc);
6910
6911 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6912 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6913 {
6914 rc = VMXLOCAL_READ_SEG(TR, tr);
6915 AssertRCReturn(rc, rc);
6916 }
6917 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6918 }
6919 return rc;
6920}
6921
6922#undef VMXLOCAL_READ_SEG
6923
6924
6925/**
6926 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6927 * context.
6928 *
6929 * @returns VBox status code.
6930 * @param pVCpu The cross context virtual CPU structure.
6931 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6932 * out-of-sync. Make sure to update the required fields
6933 * before using them.
6934 *
6935 * @remarks No-long-jump zone!!!
6936 */
6937static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6938{
6939 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6940 {
6941 if (!pVCpu->hm.s.fUsingHyperDR7)
6942 {
6943 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6944 uint32_t u32Val;
6945 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6946 pMixedCtx->dr[7] = u32Val;
6947 }
6948
6949 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6950 }
6951 return VINF_SUCCESS;
6952}
6953
6954
6955/**
6956 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6957 *
6958 * @returns VBox status code.
6959 * @param pVCpu The cross context virtual CPU structure.
6960 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6961 * out-of-sync. Make sure to update the required fields
6962 * before using them.
6963 *
6964 * @remarks No-long-jump zone!!!
6965 */
6966static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6967{
6968 NOREF(pMixedCtx);
6969
6970 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6971 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6972 return VINF_SUCCESS;
6973}
6974
6975
6976/**
6977 * Saves the entire guest state from the currently active VMCS into the
6978 * guest-CPU context.
6979 *
6980 * This essentially VMREADs all guest-data.
6981 *
6982 * @returns VBox status code.
6983 * @param pVCpu The cross context virtual CPU structure.
6984 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6985 * out-of-sync. Make sure to update the required fields
6986 * before using them.
6987 */
6988static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6989{
6990 Assert(pVCpu);
6991 Assert(pMixedCtx);
6992
6993 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6994 return VINF_SUCCESS;
6995
6996 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6997 again on the ring-3 callback path, there is no real need to. */
6998 if (VMMRZCallRing3IsEnabled(pVCpu))
6999 VMMR0LogFlushDisable(pVCpu);
7000 else
7001 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7002 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7003
7004 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7005 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7006
7007 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7008 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7009
7010 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7011 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7012
7013 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7014 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7015
7016 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7017 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7018
7019 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7020 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7021
7022 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7023 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7024
7025 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7026 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7027
7028 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7029 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7030
7031 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7032 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7033
7034 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7035 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7036 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7037
7038 if (VMMRZCallRing3IsEnabled(pVCpu))
7039 VMMR0LogFlushEnable(pVCpu);
7040
7041 return VINF_SUCCESS;
7042}
7043
7044
7045/**
7046 * Saves basic guest registers needed for IEM instruction execution.
7047 *
7048 * @returns VBox status code (OR-able).
7049 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7050 * @param pMixedCtx Pointer to the CPU context of the guest.
7051 * @param fMemory Whether the instruction being executed operates on
7052 * memory or not. Only CR0 is synced up if clear.
7053 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7054 */
7055static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7056{
7057 /*
7058 * We assume all general purpose registers other than RSP are available.
7059 *
7060 * - RIP is a must, as it will be incremented or otherwise changed.
7061 * - RFLAGS are always required to figure the CPL.
7062 * - RSP isn't always required, however it's a GPR, so frequently required.
7063 * - SS and CS are the only segment register needed if IEM doesn't do memory
7064 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7065 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7066 * be required for memory accesses.
7067 *
7068 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7069 */
7070 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7071 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7072 if (fNeedRsp)
7073 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7074 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7075 if (!fMemory)
7076 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7077 else
7078 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7079 AssertRCReturn(rc, rc);
7080 return rc;
7081}
7082
7083
7084/**
7085 * Ensures that we've got a complete basic guest-context.
7086 *
7087 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7088 * is for the interpreter.
7089 *
7090 * @returns VBox status code.
7091 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7092 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7093 * needing to be synced in.
7094 * @thread EMT(pVCpu)
7095 */
7096VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7097{
7098 /* Note! Since this is only applicable to VT-x, the implementation is placed
7099 in the VT-x part of the sources instead of the generic stuff. */
7100 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
7101 {
7102 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7103 /*
7104 * For now, imply that the caller might change everything too. Do this after
7105 * saving the guest state so as to not trigger assertions.
7106 */
7107 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7108 return rc;
7109 }
7110 return VINF_SUCCESS;
7111}
7112
7113
7114/**
7115 * Check per-VM and per-VCPU force flag actions that require us to go back to
7116 * ring-3 for one reason or another.
7117 *
7118 * @returns Strict VBox status code (i.e. informational status codes too)
7119 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7120 * ring-3.
7121 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7122 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7123 * interrupts)
7124 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7125 * all EMTs to be in ring-3.
7126 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7127 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7128 * to the EM loop.
7129 *
7130 * @param pVM The cross context VM structure.
7131 * @param pVCpu The cross context virtual CPU structure.
7132 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7133 * out-of-sync. Make sure to update the required fields
7134 * before using them.
7135 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7136 */
7137static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7138{
7139 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7140
7141 /*
7142 * Anything pending? Should be more likely than not if we're doing a good job.
7143 */
7144 if ( !fStepping
7145 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7146 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7147 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7148 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7149 return VINF_SUCCESS;
7150
7151 /* We need the control registers now, make sure the guest-CPU context is updated. */
7152 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7153 AssertRCReturn(rc3, rc3);
7154
7155 /* Pending HM CR3 sync. */
7156 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7157 {
7158 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7159 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7160 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7161 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7162 }
7163
7164 /* Pending HM PAE PDPEs. */
7165 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7166 {
7167 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7168 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7169 }
7170
7171 /* Pending PGM C3 sync. */
7172 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7173 {
7174 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7175 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7176 if (rcStrict2 != VINF_SUCCESS)
7177 {
7178 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7179 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7180 return rcStrict2;
7181 }
7182 }
7183
7184 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7185 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7186 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7187 {
7188 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7189 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7190 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7191 return rc2;
7192 }
7193
7194 /* Pending VM request packets, such as hardware interrupts. */
7195 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7196 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7197 {
7198 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7199 return VINF_EM_PENDING_REQUEST;
7200 }
7201
7202 /* Pending PGM pool flushes. */
7203 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7204 {
7205 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7206 return VINF_PGM_POOL_FLUSH_PENDING;
7207 }
7208
7209 /* Pending DMA requests. */
7210 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7211 {
7212 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7213 return VINF_EM_RAW_TO_R3;
7214 }
7215
7216 return VINF_SUCCESS;
7217}
7218
7219
7220/**
7221 * Converts any TRPM trap into a pending HM event. This is typically used when
7222 * entering from ring-3 (not longjmp returns).
7223 *
7224 * @param pVCpu The cross context virtual CPU structure.
7225 */
7226static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7227{
7228 Assert(TRPMHasTrap(pVCpu));
7229 Assert(!pVCpu->hm.s.Event.fPending);
7230
7231 uint8_t uVector;
7232 TRPMEVENT enmTrpmEvent;
7233 RTGCUINT uErrCode;
7234 RTGCUINTPTR GCPtrFaultAddress;
7235 uint8_t cbInstr;
7236
7237 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7238 AssertRC(rc);
7239
7240 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7241 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7242 if (enmTrpmEvent == TRPM_TRAP)
7243 {
7244 switch (uVector)
7245 {
7246 case X86_XCPT_NMI:
7247 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7248 break;
7249
7250 case X86_XCPT_BP:
7251 case X86_XCPT_OF:
7252 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7253 break;
7254
7255 case X86_XCPT_PF:
7256 case X86_XCPT_DF:
7257 case X86_XCPT_TS:
7258 case X86_XCPT_NP:
7259 case X86_XCPT_SS:
7260 case X86_XCPT_GP:
7261 case X86_XCPT_AC:
7262 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7263 /* fall thru */
7264 default:
7265 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7266 break;
7267 }
7268 }
7269 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7270 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7271 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7272 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7273 else
7274 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7275
7276 rc = TRPMResetTrap(pVCpu);
7277 AssertRC(rc);
7278 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7279 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7280
7281 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7282}
7283
7284
7285/**
7286 * Converts the pending HM event into a TRPM trap.
7287 *
7288 * @param pVCpu The cross context virtual CPU structure.
7289 */
7290static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7291{
7292 Assert(pVCpu->hm.s.Event.fPending);
7293
7294 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7295 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7296 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7297 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7298
7299 /* If a trap was already pending, we did something wrong! */
7300 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7301
7302 TRPMEVENT enmTrapType;
7303 switch (uVectorType)
7304 {
7305 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7306 enmTrapType = TRPM_HARDWARE_INT;
7307 break;
7308
7309 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7310 enmTrapType = TRPM_SOFTWARE_INT;
7311 break;
7312
7313 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7314 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7315 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7316 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7317 enmTrapType = TRPM_TRAP;
7318 break;
7319
7320 default:
7321 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7322 enmTrapType = TRPM_32BIT_HACK;
7323 break;
7324 }
7325
7326 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7327
7328 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7329 AssertRC(rc);
7330
7331 if (fErrorCodeValid)
7332 TRPMSetErrorCode(pVCpu, uErrorCode);
7333
7334 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7335 && uVector == X86_XCPT_PF)
7336 {
7337 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7338 }
7339 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7340 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7341 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7342 {
7343 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7344 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7345 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7346 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7347 }
7348
7349 /* Clear any pending events from the VMCS. */
7350 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7351 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7352
7353 /* We're now done converting the pending event. */
7354 pVCpu->hm.s.Event.fPending = false;
7355}
7356
7357
7358/**
7359 * Does the necessary state syncing before returning to ring-3 for any reason
7360 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7361 *
7362 * @returns VBox status code.
7363 * @param pVCpu The cross context virtual CPU structure.
7364 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7365 * be out-of-sync. Make sure to update the required
7366 * fields before using them.
7367 * @param fSaveGuestState Whether to save the guest state or not.
7368 *
7369 * @remarks No-long-jmp zone!!!
7370 */
7371static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7372{
7373 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7374 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7375
7376 RTCPUID idCpu = RTMpCpuId();
7377 Log4Func(("HostCpuId=%u\n", idCpu));
7378
7379 /*
7380 * !!! IMPORTANT !!!
7381 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7382 */
7383
7384 /* Save the guest state if necessary. */
7385 if ( fSaveGuestState
7386 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7387 {
7388 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7389 AssertRCReturn(rc, rc);
7390 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7391 }
7392
7393 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7394 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7395 {
7396 /* We shouldn't reload CR0 without saving it first. */
7397 if (!fSaveGuestState)
7398 {
7399 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7400 AssertRCReturn(rc, rc);
7401 }
7402 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7403 }
7404
7405 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7406#ifdef VBOX_STRICT
7407 if (CPUMIsHyperDebugStateActive(pVCpu))
7408 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7409#endif
7410 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7411 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7412 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7413 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7414
7415#if HC_ARCH_BITS == 64
7416 /* Restore host-state bits that VT-x only restores partially. */
7417 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7418 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7419 {
7420 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7421 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7422 }
7423 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7424#endif
7425
7426 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7427 if (pVCpu->hm.s.vmx.fLazyMsrs)
7428 {
7429 /* We shouldn't reload the guest MSRs without saving it first. */
7430 if (!fSaveGuestState)
7431 {
7432 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7433 AssertRCReturn(rc, rc);
7434 }
7435 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7436 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7437 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7438 }
7439
7440 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7441 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7442
7443 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7444 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7445 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7446 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7447 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7448 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7449 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7450 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7451
7452 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7453
7454 /** @todo This partially defeats the purpose of having preemption hooks.
7455 * The problem is, deregistering the hooks should be moved to a place that
7456 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7457 * context.
7458 */
7459 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7460 {
7461 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7462 AssertRCReturn(rc, rc);
7463
7464 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7465 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7466 }
7467 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7468 NOREF(idCpu);
7469
7470 return VINF_SUCCESS;
7471}
7472
7473
7474/**
7475 * Leaves the VT-x session.
7476 *
7477 * @returns VBox status code.
7478 * @param pVCpu The cross context virtual CPU structure.
7479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7480 * out-of-sync. Make sure to update the required fields
7481 * before using them.
7482 *
7483 * @remarks No-long-jmp zone!!!
7484 */
7485DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7486{
7487 HM_DISABLE_PREEMPT();
7488 HMVMX_ASSERT_CPU_SAFE();
7489 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7491
7492 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7493 and done this from the VMXR0ThreadCtxCallback(). */
7494 if (!pVCpu->hm.s.fLeaveDone)
7495 {
7496 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7497 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7498 pVCpu->hm.s.fLeaveDone = true;
7499 }
7500 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7501
7502 /*
7503 * !!! IMPORTANT !!!
7504 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7505 */
7506
7507 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7508 /** @todo Deregistering here means we need to VMCLEAR always
7509 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7510 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7511 VMMR0ThreadCtxHookDisable(pVCpu);
7512
7513 /* Leave HM context. This takes care of local init (term). */
7514 int rc = HMR0LeaveCpu(pVCpu);
7515
7516 HM_RESTORE_PREEMPT();
7517 return rc;
7518}
7519
7520
7521/**
7522 * Does the necessary state syncing before doing a longjmp to ring-3.
7523 *
7524 * @returns VBox status code.
7525 * @param pVCpu The cross context virtual CPU structure.
7526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7527 * out-of-sync. Make sure to update the required fields
7528 * before using them.
7529 *
7530 * @remarks No-long-jmp zone!!!
7531 */
7532DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7533{
7534 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7535}
7536
7537
7538/**
7539 * Take necessary actions before going back to ring-3.
7540 *
7541 * An action requires us to go back to ring-3. This function does the necessary
7542 * steps before we can safely return to ring-3. This is not the same as longjmps
7543 * to ring-3, this is voluntary and prepares the guest so it may continue
7544 * executing outside HM (recompiler/IEM).
7545 *
7546 * @returns VBox status code.
7547 * @param pVM The cross context VM structure.
7548 * @param pVCpu The cross context virtual CPU structure.
7549 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7550 * out-of-sync. Make sure to update the required fields
7551 * before using them.
7552 * @param rcExit The reason for exiting to ring-3. Can be
7553 * VINF_VMM_UNKNOWN_RING3_CALL.
7554 */
7555static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7556{
7557 Assert(pVM);
7558 Assert(pVCpu);
7559 Assert(pMixedCtx);
7560 HMVMX_ASSERT_PREEMPT_SAFE();
7561
7562 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7563 {
7564 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7565 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7566 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7567 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7568 }
7569
7570 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7571 VMMRZCallRing3Disable(pVCpu);
7572 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7573
7574 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7575 if (pVCpu->hm.s.Event.fPending)
7576 {
7577 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7578 Assert(!pVCpu->hm.s.Event.fPending);
7579 }
7580
7581 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7582 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7583
7584 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7585 and if we're injecting an event we should have a TRPM trap pending. */
7586 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7587#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7588 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7589#endif
7590
7591 /* Save guest state and restore host state bits. */
7592 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7593 AssertRCReturn(rc, rc);
7594 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7595 /* Thread-context hooks are unregistered at this point!!! */
7596
7597 /* Sync recompiler state. */
7598 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7599 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7600 | CPUM_CHANGED_LDTR
7601 | CPUM_CHANGED_GDTR
7602 | CPUM_CHANGED_IDTR
7603 | CPUM_CHANGED_TR
7604 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7605 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7606 if ( pVM->hm.s.fNestedPaging
7607 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7608 {
7609 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7610 }
7611
7612 Assert(!pVCpu->hm.s.fClearTrapFlag);
7613
7614 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7615 if (rcExit != VINF_EM_RAW_INTERRUPT)
7616 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7617
7618 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7619
7620 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7621 VMMRZCallRing3RemoveNotification(pVCpu);
7622 VMMRZCallRing3Enable(pVCpu);
7623
7624 return rc;
7625}
7626
7627
7628/**
7629 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7630 * longjump to ring-3 and possibly get preempted.
7631 *
7632 * @returns VBox status code.
7633 * @param pVCpu The cross context virtual CPU structure.
7634 * @param enmOperation The operation causing the ring-3 longjump.
7635 * @param pvUser Opaque pointer to the guest-CPU context. The data
7636 * may be out-of-sync. Make sure to update the required
7637 * fields before using them.
7638 */
7639static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7640{
7641 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7642 {
7643 /*
7644 * !!! IMPORTANT !!!
7645 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7646 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7647 */
7648 VMMRZCallRing3RemoveNotification(pVCpu);
7649 VMMRZCallRing3Disable(pVCpu);
7650 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7651 RTThreadPreemptDisable(&PreemptState);
7652
7653 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7654 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7655
7656#if HC_ARCH_BITS == 64
7657 /* Restore host-state bits that VT-x only restores partially. */
7658 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7659 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7660 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7661 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7662#endif
7663 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7664 if (pVCpu->hm.s.vmx.fLazyMsrs)
7665 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7666
7667 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7668 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7669 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7670 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7671 {
7672 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7673 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7674 }
7675
7676 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7677 VMMR0ThreadCtxHookDisable(pVCpu);
7678 HMR0LeaveCpu(pVCpu);
7679 RTThreadPreemptRestore(&PreemptState);
7680 return VINF_SUCCESS;
7681 }
7682
7683 Assert(pVCpu);
7684 Assert(pvUser);
7685 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7686 HMVMX_ASSERT_PREEMPT_SAFE();
7687
7688 VMMRZCallRing3Disable(pVCpu);
7689 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7690
7691 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7692 enmOperation));
7693
7694 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7695 AssertRCReturn(rc, rc);
7696
7697 VMMRZCallRing3Enable(pVCpu);
7698 return VINF_SUCCESS;
7699}
7700
7701
7702/**
7703 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7704 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7705 *
7706 * @param pVCpu The cross context virtual CPU structure.
7707 */
7708DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7709{
7710 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7711 {
7712 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7713 {
7714 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7715 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7716 AssertRC(rc);
7717 Log4(("Setup interrupt-window exiting\n"));
7718 }
7719 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7720}
7721
7722
7723/**
7724 * Clears the interrupt-window exiting control in the VMCS.
7725 *
7726 * @param pVCpu The cross context virtual CPU structure.
7727 */
7728DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7729{
7730 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7731 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7732 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7733 AssertRC(rc);
7734 Log4(("Cleared interrupt-window exiting\n"));
7735}
7736
7737
7738/**
7739 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7740 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7741 *
7742 * @param pVCpu The cross context virtual CPU structure.
7743 */
7744DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7745{
7746 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7747 {
7748 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7749 {
7750 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7751 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7752 AssertRC(rc);
7753 Log4(("Setup NMI-window exiting\n"));
7754 }
7755 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7756}
7757
7758
7759/**
7760 * Clears the NMI-window exiting control in the VMCS.
7761 *
7762 * @param pVCpu The cross context virtual CPU structure.
7763 */
7764DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7765{
7766 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7767 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7768 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7769 AssertRC(rc);
7770 Log4(("Cleared NMI-window exiting\n"));
7771}
7772
7773
7774/**
7775 * Evaluates the event to be delivered to the guest and sets it as the pending
7776 * event.
7777 *
7778 * @returns The VT-x guest-interruptibility state.
7779 * @param pVCpu The cross context virtual CPU structure.
7780 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7781 * out-of-sync. Make sure to update the required fields
7782 * before using them.
7783 */
7784static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7785{
7786 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7787 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7788 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7789 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7790 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7791
7792 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7793 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7794 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7795 Assert(!TRPMHasTrap(pVCpu));
7796
7797 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7798 APICUpdatePendingInterrupts(pVCpu);
7799
7800 /*
7801 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7802 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7803 */
7804 /** @todo SMI. SMIs take priority over NMIs. */
7805 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7806 {
7807 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7808 if ( !pVCpu->hm.s.Event.fPending
7809 && !fBlockNmi
7810 && !fBlockSti
7811 && !fBlockMovSS)
7812 {
7813 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7814 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7815 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7816
7817 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7818 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7819 }
7820 else
7821 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7822 }
7823 /*
7824 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7825 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7826 */
7827 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7828 && !pVCpu->hm.s.fSingleInstruction)
7829 {
7830 Assert(!DBGFIsStepping(pVCpu));
7831 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7832 AssertRC(rc);
7833 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7834 if ( !pVCpu->hm.s.Event.fPending
7835 && !fBlockInt
7836 && !fBlockSti
7837 && !fBlockMovSS)
7838 {
7839 uint8_t u8Interrupt;
7840 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7841 if (RT_SUCCESS(rc))
7842 {
7843 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7844 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7845 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7846
7847 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7848 }
7849 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7850 {
7851 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7852 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7853 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7854
7855 /*
7856 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7857 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7858 * need to re-set this force-flag here.
7859 */
7860 }
7861 else
7862 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7863 }
7864 else
7865 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7866 }
7867
7868 return uIntrState;
7869}
7870
7871
7872/**
7873 * Sets a pending-debug exception to be delivered to the guest if the guest is
7874 * single-stepping in the VMCS.
7875 *
7876 * @param pVCpu The cross context virtual CPU structure.
7877 */
7878DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7879{
7880 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7881 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7882 AssertRC(rc);
7883}
7884
7885
7886/**
7887 * Injects any pending events into the guest if the guest is in a state to
7888 * receive them.
7889 *
7890 * @returns Strict VBox status code (i.e. informational status codes too).
7891 * @param pVCpu The cross context virtual CPU structure.
7892 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7893 * out-of-sync. Make sure to update the required fields
7894 * before using them.
7895 * @param uIntrState The VT-x guest-interruptibility state.
7896 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7897 * return VINF_EM_DBG_STEPPED if the event was
7898 * dispatched directly.
7899 */
7900static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7901{
7902 HMVMX_ASSERT_PREEMPT_SAFE();
7903 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7904
7905 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7906 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7907
7908 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7909 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7910 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7911 Assert(!TRPMHasTrap(pVCpu));
7912
7913 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7914 if (pVCpu->hm.s.Event.fPending)
7915 {
7916 /*
7917 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7918 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7919 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7920 *
7921 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7922 */
7923 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7924#ifdef VBOX_STRICT
7925 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7926 {
7927 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7928 Assert(!fBlockInt);
7929 Assert(!fBlockSti);
7930 Assert(!fBlockMovSS);
7931 }
7932 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7933 {
7934 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7935 Assert(!fBlockSti);
7936 Assert(!fBlockMovSS);
7937 Assert(!fBlockNmi);
7938 }
7939#endif
7940 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7941 (uint8_t)uIntType));
7942 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7943 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7944 fStepping, &uIntrState);
7945 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7946
7947 /* Update the interruptibility-state as it could have been changed by
7948 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7949 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7950 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7951
7952 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7953 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7954 else
7955 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7956 }
7957
7958 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7959 if ( fBlockSti
7960 || fBlockMovSS)
7961 {
7962 if (!pVCpu->hm.s.fSingleInstruction)
7963 {
7964 /*
7965 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7966 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7967 * See Intel spec. 27.3.4 "Saving Non-Register State".
7968 */
7969 Assert(!DBGFIsStepping(pVCpu));
7970 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7971 AssertRCReturn(rc2, rc2);
7972 if (pMixedCtx->eflags.Bits.u1TF)
7973 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7974 }
7975 else if (pMixedCtx->eflags.Bits.u1TF)
7976 {
7977 /*
7978 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7979 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7980 */
7981 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7982 uIntrState = 0;
7983 }
7984 }
7985
7986 /*
7987 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7988 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7989 */
7990 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7991 AssertRC(rc2);
7992
7993 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7994 NOREF(fBlockMovSS); NOREF(fBlockSti);
7995 return rcStrict;
7996}
7997
7998
7999/**
8000 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8001 *
8002 * @param pVCpu The cross context virtual CPU structure.
8003 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8004 * out-of-sync. Make sure to update the required fields
8005 * before using them.
8006 */
8007DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8008{
8009 NOREF(pMixedCtx);
8010 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8011 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8012}
8013
8014
8015/**
8016 * Injects a double-fault (\#DF) exception into the VM.
8017 *
8018 * @returns Strict VBox status code (i.e. informational status codes too).
8019 * @param pVCpu The cross context virtual CPU structure.
8020 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8021 * out-of-sync. Make sure to update the required fields
8022 * before using them.
8023 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8024 * and should return VINF_EM_DBG_STEPPED if the event
8025 * is injected directly (register modified by us, not
8026 * by hardware on VM-entry).
8027 * @param puIntrState Pointer to the current guest interruptibility-state.
8028 * This interruptibility-state will be updated if
8029 * necessary. This cannot not be NULL.
8030 */
8031DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8032{
8033 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8034 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8035 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8036 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8037 fStepping, puIntrState);
8038}
8039
8040
8041/**
8042 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8043 *
8044 * @param pVCpu The cross context virtual CPU structure.
8045 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8046 * out-of-sync. Make sure to update the required fields
8047 * before using them.
8048 */
8049DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8050{
8051 NOREF(pMixedCtx);
8052 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8053 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8054 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8055}
8056
8057
8058/**
8059 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8060 *
8061 * @param pVCpu The cross context virtual CPU structure.
8062 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8063 * out-of-sync. Make sure to update the required fields
8064 * before using them.
8065 * @param cbInstr The value of RIP that is to be pushed on the guest
8066 * stack.
8067 */
8068DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8069{
8070 NOREF(pMixedCtx);
8071 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8072 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8073 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8074}
8075
8076
8077/**
8078 * Injects a general-protection (\#GP) fault into the VM.
8079 *
8080 * @returns Strict VBox status code (i.e. informational status codes too).
8081 * @param pVCpu The cross context virtual CPU structure.
8082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8083 * out-of-sync. Make sure to update the required fields
8084 * before using them.
8085 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8086 * mode, i.e. in real-mode it's not valid).
8087 * @param u32ErrorCode The error code associated with the \#GP.
8088 * @param fStepping Whether we're running in
8089 * hmR0VmxRunGuestCodeStep() and should return
8090 * VINF_EM_DBG_STEPPED if the event is injected
8091 * directly (register modified by us, not by
8092 * hardware on VM-entry).
8093 * @param puIntrState Pointer to the current guest interruptibility-state.
8094 * This interruptibility-state will be updated if
8095 * necessary. This cannot not be NULL.
8096 */
8097DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8098 bool fStepping, uint32_t *puIntrState)
8099{
8100 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8101 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8102 if (fErrorCodeValid)
8103 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8104 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8105 fStepping, puIntrState);
8106}
8107
8108
8109#if 0 /* unused */
8110/**
8111 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8112 * VM.
8113 *
8114 * @param pVCpu The cross context virtual CPU structure.
8115 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8116 * out-of-sync. Make sure to update the required fields
8117 * before using them.
8118 * @param u32ErrorCode The error code associated with the \#GP.
8119 */
8120DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8121{
8122 NOREF(pMixedCtx);
8123 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8124 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8125 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8126 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8127}
8128#endif /* unused */
8129
8130
8131/**
8132 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8133 *
8134 * @param pVCpu The cross context virtual CPU structure.
8135 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8136 * out-of-sync. Make sure to update the required fields
8137 * before using them.
8138 * @param uVector The software interrupt vector number.
8139 * @param cbInstr The value of RIP that is to be pushed on the guest
8140 * stack.
8141 */
8142DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8143{
8144 NOREF(pMixedCtx);
8145 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8146 if ( uVector == X86_XCPT_BP
8147 || uVector == X86_XCPT_OF)
8148 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8149 else
8150 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8151 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8152}
8153
8154
8155/**
8156 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8157 * stack.
8158 *
8159 * @returns Strict VBox status code (i.e. informational status codes too).
8160 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8161 * @param pVM The cross context VM structure.
8162 * @param pMixedCtx Pointer to the guest-CPU context.
8163 * @param uValue The value to push to the guest stack.
8164 */
8165DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8166{
8167 /*
8168 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8169 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8170 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8171 */
8172 if (pMixedCtx->sp == 1)
8173 return VINF_EM_RESET;
8174 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8175 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8176 AssertRC(rc);
8177 return rc;
8178}
8179
8180
8181/**
8182 * Injects an event into the guest upon VM-entry by updating the relevant fields
8183 * in the VM-entry area in the VMCS.
8184 *
8185 * @returns Strict VBox status code (i.e. informational status codes too).
8186 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8187 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8188 *
8189 * @param pVCpu The cross context virtual CPU structure.
8190 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8191 * be out-of-sync. Make sure to update the required
8192 * fields before using them.
8193 * @param u64IntInfo The VM-entry interruption-information field.
8194 * @param cbInstr The VM-entry instruction length in bytes (for
8195 * software interrupts, exceptions and privileged
8196 * software exceptions).
8197 * @param u32ErrCode The VM-entry exception error code.
8198 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8199 * @param puIntrState Pointer to the current guest interruptibility-state.
8200 * This interruptibility-state will be updated if
8201 * necessary. This cannot not be NULL.
8202 * @param fStepping Whether we're running in
8203 * hmR0VmxRunGuestCodeStep() and should return
8204 * VINF_EM_DBG_STEPPED if the event is injected
8205 * directly (register modified by us, not by
8206 * hardware on VM-entry).
8207 *
8208 * @remarks Requires CR0!
8209 */
8210static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8211 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8212 uint32_t *puIntrState)
8213{
8214 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8215 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8216 Assert(puIntrState);
8217 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8218
8219 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8220 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8221
8222#ifdef VBOX_STRICT
8223 /* Validate the error-code-valid bit for hardware exceptions. */
8224 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8225 {
8226 switch (uVector)
8227 {
8228 case X86_XCPT_PF:
8229 case X86_XCPT_DF:
8230 case X86_XCPT_TS:
8231 case X86_XCPT_NP:
8232 case X86_XCPT_SS:
8233 case X86_XCPT_GP:
8234 case X86_XCPT_AC:
8235 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8236 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8237 /* fall thru */
8238 default:
8239 break;
8240 }
8241 }
8242#endif
8243
8244 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8245 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8246 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8247
8248 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8249
8250 /* We require CR0 to check if the guest is in real-mode. */
8251 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8252 AssertRCReturn(rc, rc);
8253
8254 /*
8255 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8256 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8257 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8258 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8259 */
8260 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8261 {
8262 PVM pVM = pVCpu->CTX_SUFF(pVM);
8263 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8264 {
8265 Assert(PDMVmmDevHeapIsEnabled(pVM));
8266 Assert(pVM->hm.s.vmx.pRealModeTSS);
8267
8268 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8269 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8270 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8271 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8272 AssertRCReturn(rc, rc);
8273 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8274
8275 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8276 size_t const cbIdtEntry = sizeof(X86IDTR16);
8277 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8278 {
8279 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8280 if (uVector == X86_XCPT_DF)
8281 return VINF_EM_RESET;
8282
8283 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8284 if (uVector == X86_XCPT_GP)
8285 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8286
8287 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8288 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8289 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8290 fStepping, puIntrState);
8291 }
8292
8293 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8294 uint16_t uGuestIp = pMixedCtx->ip;
8295 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8296 {
8297 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8298 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8299 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8300 }
8301 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8302 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8303
8304 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8305 X86IDTR16 IdtEntry;
8306 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8307 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8308 AssertRCReturn(rc, rc);
8309
8310 /* Construct the stack frame for the interrupt/exception handler. */
8311 VBOXSTRICTRC rcStrict;
8312 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8313 if (rcStrict == VINF_SUCCESS)
8314 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8315 if (rcStrict == VINF_SUCCESS)
8316 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8317
8318 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8319 if (rcStrict == VINF_SUCCESS)
8320 {
8321 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8322 pMixedCtx->rip = IdtEntry.offSel;
8323 pMixedCtx->cs.Sel = IdtEntry.uSel;
8324 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8325 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8326 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8327 && uVector == X86_XCPT_PF)
8328 pMixedCtx->cr2 = GCPtrFaultAddress;
8329
8330 /* If any other guest-state bits are changed here, make sure to update
8331 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8332 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8333 | HM_CHANGED_GUEST_RIP
8334 | HM_CHANGED_GUEST_RFLAGS
8335 | HM_CHANGED_GUEST_RSP);
8336
8337 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8338 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8339 {
8340 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8341 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8342 Log4(("Clearing inhibition due to STI.\n"));
8343 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8344 }
8345 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8346 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8347
8348 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8349 it, if we are returning to ring-3 before executing guest code. */
8350 pVCpu->hm.s.Event.fPending = false;
8351
8352 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8353 if (fStepping)
8354 rcStrict = VINF_EM_DBG_STEPPED;
8355 }
8356 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8357 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8358 return rcStrict;
8359 }
8360
8361 /*
8362 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8363 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8364 */
8365 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8366 }
8367
8368 /* Validate. */
8369 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8370 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8371 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8372
8373 /* Inject. */
8374 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8375 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8376 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8377 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8378
8379 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8380 && uVector == X86_XCPT_PF)
8381 pMixedCtx->cr2 = GCPtrFaultAddress;
8382
8383 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8384 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8385
8386 AssertRCReturn(rc, rc);
8387 return VINF_SUCCESS;
8388}
8389
8390
8391/**
8392 * Clears the interrupt-window exiting control in the VMCS and if necessary
8393 * clears the current event in the VMCS as well.
8394 *
8395 * @returns VBox status code.
8396 * @param pVCpu The cross context virtual CPU structure.
8397 *
8398 * @remarks Use this function only to clear events that have not yet been
8399 * delivered to the guest but are injected in the VMCS!
8400 * @remarks No-long-jump zone!!!
8401 */
8402static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8403{
8404 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8405
8406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8407 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8408
8409 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8410 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8411}
8412
8413
8414/**
8415 * Enters the VT-x session.
8416 *
8417 * @returns VBox status code.
8418 * @param pVM The cross context VM structure.
8419 * @param pVCpu The cross context virtual CPU structure.
8420 * @param pCpu Pointer to the CPU info struct.
8421 */
8422VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8423{
8424 AssertPtr(pVM);
8425 AssertPtr(pVCpu);
8426 Assert(pVM->hm.s.vmx.fSupported);
8427 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8428 NOREF(pCpu); NOREF(pVM);
8429
8430 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8431 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8432
8433#ifdef VBOX_STRICT
8434 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8435 RTCCUINTREG uHostCR4 = ASMGetCR4();
8436 if (!(uHostCR4 & X86_CR4_VMXE))
8437 {
8438 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8439 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8440 }
8441#endif
8442
8443 /*
8444 * Load the VCPU's VMCS as the current (and active) one.
8445 */
8446 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8447 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8448 if (RT_FAILURE(rc))
8449 return rc;
8450
8451 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8452 pVCpu->hm.s.fLeaveDone = false;
8453 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8454
8455 return VINF_SUCCESS;
8456}
8457
8458
8459/**
8460 * The thread-context callback (only on platforms which support it).
8461 *
8462 * @param enmEvent The thread-context event.
8463 * @param pVCpu The cross context virtual CPU structure.
8464 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8465 * @thread EMT(pVCpu)
8466 */
8467VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8468{
8469 NOREF(fGlobalInit);
8470
8471 switch (enmEvent)
8472 {
8473 case RTTHREADCTXEVENT_OUT:
8474 {
8475 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8476 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8477 VMCPU_ASSERT_EMT(pVCpu);
8478
8479 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8480
8481 /* No longjmps (logger flushes, locks) in this fragile context. */
8482 VMMRZCallRing3Disable(pVCpu);
8483 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8484
8485 /*
8486 * Restore host-state (FPU, debug etc.)
8487 */
8488 if (!pVCpu->hm.s.fLeaveDone)
8489 {
8490 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8491 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8492 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8493 pVCpu->hm.s.fLeaveDone = true;
8494 }
8495
8496 /* Leave HM context, takes care of local init (term). */
8497 int rc = HMR0LeaveCpu(pVCpu);
8498 AssertRC(rc); NOREF(rc);
8499
8500 /* Restore longjmp state. */
8501 VMMRZCallRing3Enable(pVCpu);
8502 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8503 break;
8504 }
8505
8506 case RTTHREADCTXEVENT_IN:
8507 {
8508 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8509 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8510 VMCPU_ASSERT_EMT(pVCpu);
8511
8512 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8513 VMMRZCallRing3Disable(pVCpu);
8514 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8515
8516 /* Initialize the bare minimum state required for HM. This takes care of
8517 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8518 int rc = HMR0EnterCpu(pVCpu);
8519 AssertRC(rc);
8520 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8521
8522 /* Load the active VMCS as the current one. */
8523 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8524 {
8525 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8526 AssertRC(rc); NOREF(rc);
8527 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8528 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8529 }
8530 pVCpu->hm.s.fLeaveDone = false;
8531
8532 /* Restore longjmp state. */
8533 VMMRZCallRing3Enable(pVCpu);
8534 break;
8535 }
8536
8537 default:
8538 break;
8539 }
8540}
8541
8542
8543/**
8544 * Saves the host state in the VMCS host-state.
8545 * Sets up the VM-exit MSR-load area.
8546 *
8547 * The CPU state will be loaded from these fields on every successful VM-exit.
8548 *
8549 * @returns VBox status code.
8550 * @param pVM The cross context VM structure.
8551 * @param pVCpu The cross context virtual CPU structure.
8552 *
8553 * @remarks No-long-jump zone!!!
8554 */
8555static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8556{
8557 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8558
8559 int rc = VINF_SUCCESS;
8560 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8561 {
8562 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8563 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8564
8565 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8566 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8567
8568 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8569 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8570
8571 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8572 }
8573 return rc;
8574}
8575
8576
8577/**
8578 * Saves the host state in the VMCS host-state.
8579 *
8580 * @returns VBox status code.
8581 * @param pVM The cross context VM structure.
8582 * @param pVCpu The cross context virtual CPU structure.
8583 *
8584 * @remarks No-long-jump zone!!!
8585 */
8586VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8587{
8588 AssertPtr(pVM);
8589 AssertPtr(pVCpu);
8590
8591 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8592
8593 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8594 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8595 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8596 return hmR0VmxSaveHostState(pVM, pVCpu);
8597}
8598
8599
8600/**
8601 * Loads the guest state into the VMCS guest-state area.
8602 *
8603 * The will typically be done before VM-entry when the guest-CPU state and the
8604 * VMCS state may potentially be out of sync.
8605 *
8606 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8607 * VM-entry controls.
8608 * Sets up the appropriate VMX non-root function to execute guest code based on
8609 * the guest CPU mode.
8610 *
8611 * @returns VBox strict status code.
8612 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8613 * without unrestricted guest access and the VMMDev is not presently
8614 * mapped (e.g. EFI32).
8615 *
8616 * @param pVM The cross context VM structure.
8617 * @param pVCpu The cross context virtual CPU structure.
8618 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8619 * out-of-sync. Make sure to update the required fields
8620 * before using them.
8621 *
8622 * @remarks No-long-jump zone!!!
8623 */
8624static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8625{
8626 AssertPtr(pVM);
8627 AssertPtr(pVCpu);
8628 AssertPtr(pMixedCtx);
8629 HMVMX_ASSERT_PREEMPT_SAFE();
8630
8631 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8632
8633 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8634
8635 /* Determine real-on-v86 mode. */
8636 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8637 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8638 && CPUMIsGuestInRealModeEx(pMixedCtx))
8639 {
8640 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8641 }
8642
8643 /*
8644 * Load the guest-state into the VMCS.
8645 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8646 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8647 */
8648 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8649 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8650
8651 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8652 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8653 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8654
8655 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8656 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8657 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8658
8659 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8660 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8661
8662 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8663 if (rcStrict == VINF_SUCCESS)
8664 { /* likely */ }
8665 else
8666 {
8667 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8668 return rcStrict;
8669 }
8670
8671 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8672 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8673 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8674
8675 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8676 determine we don't have to swap EFER after all. */
8677 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8678 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8679
8680 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8682
8683 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8685
8686 /*
8687 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8688 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8689 */
8690 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8692
8693 /* Clear any unused and reserved bits. */
8694 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8695
8696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8697 return rc;
8698}
8699
8700
8701/**
8702 * Loads the state shared between the host and guest into the VMCS.
8703 *
8704 * @param pVM The cross context VM structure.
8705 * @param pVCpu The cross context virtual CPU structure.
8706 * @param pCtx Pointer to the guest-CPU context.
8707 *
8708 * @remarks No-long-jump zone!!!
8709 */
8710static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8711{
8712 NOREF(pVM);
8713
8714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8715 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8716
8717 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8718 {
8719 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8720 AssertRC(rc);
8721 }
8722
8723 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8724 {
8725 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8726 AssertRC(rc);
8727
8728 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8729 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8730 {
8731 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8732 AssertRC(rc);
8733 }
8734 }
8735
8736 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8737 {
8738 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8739 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8740 }
8741
8742 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8744 {
8745 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8746 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8747 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8748 AssertRC(rc);
8749 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8750 }
8751
8752 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8753 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8754}
8755
8756
8757/**
8758 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8759 *
8760 * @returns Strict VBox status code (i.e. informational status codes too).
8761 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8762 * without unrestricted guest access and the VMMDev is not presently
8763 * mapped (e.g. EFI32).
8764 *
8765 * @param pVM The cross context VM structure.
8766 * @param pVCpu The cross context virtual CPU structure.
8767 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8768 * out-of-sync. Make sure to update the required fields
8769 * before using them.
8770 *
8771 * @remarks No-long-jump zone!!!
8772 */
8773static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8774{
8775 HMVMX_ASSERT_PREEMPT_SAFE();
8776 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8777 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8778
8779 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8780#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8781 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8782#endif
8783
8784 /*
8785 * RIP is what changes the most often and hence if it's the only bit needing to be
8786 * updated, we shall handle it early for performance reasons.
8787 */
8788 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8789 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8790 {
8791 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8792 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8793 { /* likely */}
8794 else
8795 {
8796 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8797 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8798 }
8799 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8800 }
8801 else if (HMCPU_CF_VALUE(pVCpu))
8802 {
8803 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8804 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8805 { /* likely */}
8806 else
8807 {
8808 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8809 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8810 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8811 return rcStrict;
8812 }
8813 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8814 }
8815
8816 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8817 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8818 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8819 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8820 return rcStrict;
8821}
8822
8823
8824/**
8825 * Does the preparations before executing guest code in VT-x.
8826 *
8827 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8828 * recompiler/IEM. We must be cautious what we do here regarding committing
8829 * guest-state information into the VMCS assuming we assuredly execute the
8830 * guest in VT-x mode.
8831 *
8832 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8833 * the common-state (TRPM/forceflags), we must undo those changes so that the
8834 * recompiler/IEM can (and should) use them when it resumes guest execution.
8835 * Otherwise such operations must be done when we can no longer exit to ring-3.
8836 *
8837 * @returns Strict VBox status code (i.e. informational status codes too).
8838 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8839 * have been disabled.
8840 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8841 * double-fault into the guest.
8842 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8843 * dispatched directly.
8844 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8845 *
8846 * @param pVM The cross context VM structure.
8847 * @param pVCpu The cross context virtual CPU structure.
8848 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8849 * out-of-sync. Make sure to update the required fields
8850 * before using them.
8851 * @param pVmxTransient Pointer to the VMX transient structure.
8852 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8853 * us ignore some of the reasons for returning to
8854 * ring-3, and return VINF_EM_DBG_STEPPED if event
8855 * dispatching took place.
8856 */
8857static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8858{
8859 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8860
8861#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8862 PGMRZDynMapFlushAutoSet(pVCpu);
8863#endif
8864
8865 /* Check force flag actions that might require us to go back to ring-3. */
8866 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8867 if (rcStrict == VINF_SUCCESS)
8868 { /* FFs doesn't get set all the time. */ }
8869 else
8870 return rcStrict;
8871
8872#ifndef IEM_VERIFICATION_MODE_FULL
8873 /*
8874 * Setup the virtualized-APIC accesses.
8875 *
8876 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8877 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8878 *
8879 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8880 */
8881 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8882 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8883 && PDMHasApic(pVM))
8884 {
8885 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8886 Assert(u64MsrApicBase);
8887 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8888
8889 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8890
8891 /* Unalias any existing mapping. */
8892 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8893 AssertRCReturn(rc, rc);
8894
8895 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8896 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8897 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8898 AssertRCReturn(rc, rc);
8899
8900 /* Update the per-VCPU cache of the APIC base MSR. */
8901 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8902 }
8903#endif /* !IEM_VERIFICATION_MODE_FULL */
8904
8905 if (TRPMHasTrap(pVCpu))
8906 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8907 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8908
8909 /*
8910 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8911 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8912 */
8913 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8914 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8915 { /* likely */ }
8916 else
8917 {
8918 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8919 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8920 return rcStrict;
8921 }
8922
8923 /*
8924 * No longjmps to ring-3 from this point on!!!
8925 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8926 * This also disables flushing of the R0-logger instance (if any).
8927 */
8928 VMMRZCallRing3Disable(pVCpu);
8929
8930 /*
8931 * Load the guest state bits.
8932 *
8933 * We cannot perform longjmps while loading the guest state because we do not preserve the
8934 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8935 * CPU migration.
8936 *
8937 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8938 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8939 * Hence, loading of the guest state needs to be done -after- injection of events.
8940 */
8941 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8942 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8943 { /* likely */ }
8944 else
8945 {
8946 VMMRZCallRing3Enable(pVCpu);
8947 return rcStrict;
8948 }
8949
8950 /*
8951 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8952 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8953 *
8954 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8955 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8956 *
8957 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8958 * executing guest code.
8959 */
8960 pVmxTransient->fEFlags = ASMIntDisableFlags();
8961
8962 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8963 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8964 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8965 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8966 {
8967 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8968 {
8969 pVCpu->hm.s.Event.fPending = false;
8970
8971 /*
8972 * We've injected any pending events. This is really the point of no return (to ring-3).
8973 *
8974 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8975 * returns from this function, so don't enable them here.
8976 */
8977 return VINF_SUCCESS;
8978 }
8979
8980 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8981 rcStrict = VINF_EM_RAW_INTERRUPT;
8982 }
8983 else
8984 {
8985 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8986 rcStrict = VINF_EM_RAW_TO_R3;
8987 }
8988
8989 ASMSetFlags(pVmxTransient->fEFlags);
8990 VMMRZCallRing3Enable(pVCpu);
8991
8992 return rcStrict;
8993}
8994
8995
8996/**
8997 * Prepares to run guest code in VT-x and we've committed to doing so. This
8998 * means there is no backing out to ring-3 or anywhere else at this
8999 * point.
9000 *
9001 * @param pVM The cross context VM structure.
9002 * @param pVCpu The cross context virtual CPU structure.
9003 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9004 * out-of-sync. Make sure to update the required fields
9005 * before using them.
9006 * @param pVmxTransient Pointer to the VMX transient structure.
9007 *
9008 * @remarks Called with preemption disabled.
9009 * @remarks No-long-jump zone!!!
9010 */
9011static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9012{
9013 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9014 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9015 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9016
9017 /*
9018 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9019 */
9020 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9021 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9022
9023#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9024 if (!CPUMIsGuestFPUStateActive(pVCpu))
9025 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9026 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9027 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9028#endif
9029
9030 if ( pVCpu->hm.s.fPreloadGuestFpu
9031 && !CPUMIsGuestFPUStateActive(pVCpu))
9032 {
9033 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9034 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9035 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9036 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9037 }
9038
9039 /*
9040 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9041 */
9042 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9043 && pVCpu->hm.s.vmx.cMsrs > 0)
9044 {
9045 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9046 }
9047
9048 /*
9049 * Load the host state bits as we may've been preempted (only happens when
9050 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9051 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9052 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9053 * See @bugref{8432}.
9054 */
9055 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9056 {
9057 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9058 AssertRC(rc);
9059 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9060 }
9061 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9062
9063 /*
9064 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9065 */
9066 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9067 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9068 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9069
9070 /* Store status of the shared guest-host state at the time of VM-entry. */
9071#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9072 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9073 {
9074 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9075 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9076 }
9077 else
9078#endif
9079 {
9080 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9081 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9082 }
9083 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
9084
9085 /*
9086 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9087 */
9088 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9089 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9090
9091 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9092 RTCPUID idCurrentCpu = pCpu->idCpu;
9093 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9094 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9095 {
9096 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9097 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9098 }
9099
9100 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9101 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9102 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9103 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9104
9105 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9106
9107 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9108 to start executing. */
9109
9110 /*
9111 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9112 */
9113 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9114 {
9115 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9116 {
9117 bool fMsrUpdated;
9118 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9119 AssertRC(rc2);
9120 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9121
9122 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9123 &fMsrUpdated);
9124 AssertRC(rc2);
9125 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9126
9127 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9128 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9129 }
9130 else
9131 {
9132 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9133 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9134 }
9135 }
9136
9137#ifdef VBOX_STRICT
9138 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9139 hmR0VmxCheckHostEferMsr(pVCpu);
9140 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9141#endif
9142#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9143 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9144 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9145 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9146#endif
9147}
9148
9149
9150/**
9151 * Performs some essential restoration of state after running guest code in
9152 * VT-x.
9153 *
9154 * @param pVM The cross context VM structure.
9155 * @param pVCpu The cross context virtual CPU structure.
9156 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9157 * out-of-sync. Make sure to update the required fields
9158 * before using them.
9159 * @param pVmxTransient Pointer to the VMX transient structure.
9160 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9161 *
9162 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9163 *
9164 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9165 * unconditionally when it is safe to do so.
9166 */
9167static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9168{
9169 NOREF(pVM);
9170
9171 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9172
9173 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9174 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9175 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9176 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9177 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9178 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9179
9180 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9181 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9182
9183 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9184 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9185 Assert(!ASMIntAreEnabled());
9186 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9187
9188#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9189 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9190 {
9191 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9192 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9193 }
9194#endif
9195
9196#if HC_ARCH_BITS == 64
9197 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9198#endif
9199#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9200 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9201 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9202 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9203#else
9204 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9205#endif
9206#ifdef VBOX_STRICT
9207 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9208#endif
9209 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9210 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
9211
9212 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9213 uint32_t uExitReason;
9214 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9215 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9216 AssertRC(rc);
9217 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9218 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9219
9220 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
9221 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
9222 {
9223 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9224 pVmxTransient->fVMEntryFailed));
9225 return;
9226 }
9227
9228 /*
9229 * Update the VM-exit history array here even if the VM-entry failed due to:
9230 * - Invalid guest state.
9231 * - MSR loading.
9232 * - Machine-check event.
9233 *
9234 * In any of the above cases we will still have a "valid" VM-exit reason
9235 * despite @a fVMEntryFailed being false.
9236 *
9237 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9238 */
9239 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9240
9241 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
9242 {
9243 /** @todo We can optimize this by only syncing with our force-flags when
9244 * really needed and keeping the VMCS state as it is for most
9245 * VM-exits. */
9246 /* Update the guest interruptibility-state from the VMCS. */
9247 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9248
9249#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9250 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9251 AssertRC(rc);
9252#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9253 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9254 AssertRC(rc);
9255#endif
9256
9257 /*
9258 * Sync the TPR shadow with our APIC state.
9259 */
9260 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9261 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9262 {
9263 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9264 AssertRC(rc);
9265 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9266 }
9267 }
9268}
9269
9270
9271/**
9272 * Runs the guest code using VT-x the normal way.
9273 *
9274 * @returns VBox status code.
9275 * @param pVM The cross context VM structure.
9276 * @param pVCpu The cross context virtual CPU structure.
9277 * @param pCtx Pointer to the guest-CPU context.
9278 *
9279 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9280 */
9281static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9282{
9283 VMXTRANSIENT VmxTransient;
9284 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9285 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9286 uint32_t cLoops = 0;
9287
9288 for (;; cLoops++)
9289 {
9290 Assert(!HMR0SuspendPending());
9291 HMVMX_ASSERT_CPU_SAFE();
9292
9293 /* Preparatory work for running guest code, this may force us to return
9294 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9295 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9296 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9297 if (rcStrict != VINF_SUCCESS)
9298 break;
9299
9300 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9301 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9302 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9303
9304 /* Restore any residual host-state and save any bits shared between host
9305 and guest into the guest-CPU state. Re-enables interrupts! */
9306 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9307
9308 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9309 if (RT_SUCCESS(rcRun))
9310 { /* very likely */ }
9311 else
9312 {
9313 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9314 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9315 return rcRun;
9316 }
9317
9318 /* Profile the VM-exit. */
9319 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9321 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9322 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9323 HMVMX_START_EXIT_DISPATCH_PROF();
9324
9325 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9326
9327 /* Handle the VM-exit. */
9328#ifdef HMVMX_USE_FUNCTION_TABLE
9329 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9330#else
9331 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9332#endif
9333 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9334 if (rcStrict == VINF_SUCCESS)
9335 {
9336 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9337 continue; /* likely */
9338 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9339 rcStrict = VINF_EM_RAW_INTERRUPT;
9340 }
9341 break;
9342 }
9343
9344 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9345 return rcStrict;
9346}
9347
9348
9349
9350/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9351 * probes.
9352 *
9353 * The following few functions and associated structure contains the bloat
9354 * necessary for providing detailed debug events and dtrace probes as well as
9355 * reliable host side single stepping. This works on the principle of
9356 * "subclassing" the normal execution loop and workers. We replace the loop
9357 * method completely and override selected helpers to add necessary adjustments
9358 * to their core operation.
9359 *
9360 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9361 * any performance for debug and analysis features.
9362 *
9363 * @{
9364 */
9365
9366/**
9367 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9368 * the debug run loop.
9369 */
9370typedef struct VMXRUNDBGSTATE
9371{
9372 /** The RIP we started executing at. This is for detecting that we stepped. */
9373 uint64_t uRipStart;
9374 /** The CS we started executing with. */
9375 uint16_t uCsStart;
9376
9377 /** Whether we've actually modified the 1st execution control field. */
9378 bool fModifiedProcCtls : 1;
9379 /** Whether we've actually modified the 2nd execution control field. */
9380 bool fModifiedProcCtls2 : 1;
9381 /** Whether we've actually modified the exception bitmap. */
9382 bool fModifiedXcptBitmap : 1;
9383
9384 /** We desire the modified the CR0 mask to be cleared. */
9385 bool fClearCr0Mask : 1;
9386 /** We desire the modified the CR4 mask to be cleared. */
9387 bool fClearCr4Mask : 1;
9388 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9389 uint32_t fCpe1Extra;
9390 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9391 uint32_t fCpe1Unwanted;
9392 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9393 uint32_t fCpe2Extra;
9394 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9395 uint32_t bmXcptExtra;
9396 /** The sequence number of the Dtrace provider settings the state was
9397 * configured against. */
9398 uint32_t uDtraceSettingsSeqNo;
9399 /** VM-exits to check (one bit per VM-exit). */
9400 uint32_t bmExitsToCheck[3];
9401
9402 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9403 uint32_t fProcCtlsInitial;
9404 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9405 uint32_t fProcCtls2Initial;
9406 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9407 uint32_t bmXcptInitial;
9408} VMXRUNDBGSTATE;
9409AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9410typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9411
9412
9413/**
9414 * Initializes the VMXRUNDBGSTATE structure.
9415 *
9416 * @param pVCpu The cross context virtual CPU structure of the
9417 * calling EMT.
9418 * @param pCtx The CPU register context to go with @a pVCpu.
9419 * @param pDbgState The structure to initialize.
9420 */
9421DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9422{
9423 pDbgState->uRipStart = pCtx->rip;
9424 pDbgState->uCsStart = pCtx->cs.Sel;
9425
9426 pDbgState->fModifiedProcCtls = false;
9427 pDbgState->fModifiedProcCtls2 = false;
9428 pDbgState->fModifiedXcptBitmap = false;
9429 pDbgState->fClearCr0Mask = false;
9430 pDbgState->fClearCr4Mask = false;
9431 pDbgState->fCpe1Extra = 0;
9432 pDbgState->fCpe1Unwanted = 0;
9433 pDbgState->fCpe2Extra = 0;
9434 pDbgState->bmXcptExtra = 0;
9435 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9436 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9437 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9438}
9439
9440
9441/**
9442 * Updates the VMSC fields with changes requested by @a pDbgState.
9443 *
9444 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9445 * immediately before executing guest code, i.e. when interrupts are disabled.
9446 * We don't check status codes here as we cannot easily assert or return in the
9447 * latter case.
9448 *
9449 * @param pVCpu The cross context virtual CPU structure.
9450 * @param pDbgState The debug state.
9451 */
9452DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9453{
9454 /*
9455 * Ensure desired flags in VMCS control fields are set.
9456 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9457 *
9458 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9459 * there should be no stale data in pCtx at this point.
9460 */
9461 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9462 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9463 {
9464 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9465 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9466 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9467 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9468 pDbgState->fModifiedProcCtls = true;
9469 }
9470
9471 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9472 {
9473 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9474 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9475 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9476 pDbgState->fModifiedProcCtls2 = true;
9477 }
9478
9479 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9480 {
9481 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9482 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9483 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9484 pDbgState->fModifiedXcptBitmap = true;
9485 }
9486
9487 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9488 {
9489 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9490 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9491 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9492 }
9493
9494 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9495 {
9496 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9497 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9498 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9499 }
9500}
9501
9502
9503DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9504{
9505 /*
9506 * Restore VM-exit control settings as we may not reenter this function the
9507 * next time around.
9508 */
9509 /* We reload the initial value, trigger what we can of recalculations the
9510 next time around. From the looks of things, that's all that's required atm. */
9511 if (pDbgState->fModifiedProcCtls)
9512 {
9513 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9514 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9515 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9516 AssertRCReturn(rc2, rc2);
9517 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9518 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9519 }
9520
9521 /* We're currently the only ones messing with this one, so just restore the
9522 cached value and reload the field. */
9523 if ( pDbgState->fModifiedProcCtls2
9524 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9525 {
9526 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9527 AssertRCReturn(rc2, rc2);
9528 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9529 }
9530
9531 /* If we've modified the exception bitmap, we restore it and trigger
9532 reloading and partial recalculation the next time around. */
9533 if (pDbgState->fModifiedXcptBitmap)
9534 {
9535 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9536 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9537 }
9538
9539 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9540 if (pDbgState->fClearCr0Mask)
9541 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9542
9543 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9544 if (pDbgState->fClearCr4Mask)
9545 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9546
9547 return rcStrict;
9548}
9549
9550
9551/**
9552 * Configures VM-exit controls for current DBGF and DTrace settings.
9553 *
9554 * This updates @a pDbgState and the VMCS execution control fields to reflect
9555 * the necessary VM-exits demanded by DBGF and DTrace.
9556 *
9557 * @param pVM The cross context VM structure.
9558 * @param pVCpu The cross context virtual CPU structure.
9559 * @param pCtx Pointer to the guest-CPU context.
9560 * @param pDbgState The debug state.
9561 * @param pVmxTransient Pointer to the VMX transient structure. May update
9562 * fUpdateTscOffsettingAndPreemptTimer.
9563 */
9564static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9565 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9566{
9567 /*
9568 * Take down the dtrace serial number so we can spot changes.
9569 */
9570 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9571 ASMCompilerBarrier();
9572
9573 /*
9574 * We'll rebuild most of the middle block of data members (holding the
9575 * current settings) as we go along here, so start by clearing it all.
9576 */
9577 pDbgState->bmXcptExtra = 0;
9578 pDbgState->fCpe1Extra = 0;
9579 pDbgState->fCpe1Unwanted = 0;
9580 pDbgState->fCpe2Extra = 0;
9581 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9582 pDbgState->bmExitsToCheck[i] = 0;
9583
9584 /*
9585 * Software interrupts (INT XXh) - no idea how to trigger these...
9586 */
9587 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9588 || VBOXVMM_INT_SOFTWARE_ENABLED())
9589 {
9590 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9591 }
9592
9593 /*
9594 * INT3 breakpoints - triggered by #BP exceptions.
9595 */
9596 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9597 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9598
9599 /*
9600 * Exception bitmap and XCPT events+probes.
9601 */
9602 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9603 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9604 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9605
9606 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9607 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9608 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9609 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9610 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9611 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9612 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9613 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9614 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9615 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9616 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9617 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9618 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9619 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9620 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9621 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9622 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9623 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9624
9625 if (pDbgState->bmXcptExtra)
9626 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9627
9628 /*
9629 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9630 *
9631 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9632 * So, when adding/changing/removing please don't forget to update it.
9633 *
9634 * Some of the macros are picking up local variables to save horizontal space,
9635 * (being able to see it in a table is the lesser evil here).
9636 */
9637#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9638 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9639 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9640#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9641 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9642 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9643 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9644 } else do { } while (0)
9645#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9646 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9647 { \
9648 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9649 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9650 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9651 } else do { } while (0)
9652#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9653 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9654 { \
9655 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9656 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9657 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9658 } else do { } while (0)
9659#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9660 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9661 { \
9662 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9663 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9664 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9665 } else do { } while (0)
9666
9667 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9668 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9669 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9670 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9671 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9672
9673 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9674 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9675 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9676 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9677 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9678 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9679 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9680 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9681 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9682 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9683 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9684 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9685 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9686 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9687 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9688 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9689 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9690 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9691 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9692 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9693 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9694 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9695 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9696 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9697 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9698 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9699 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9700 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9701 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9702 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9703 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9704 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9705 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9706 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9707 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9708 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9709
9710 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9711 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9712 {
9713 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9714 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9715 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9716 AssertRC(rc2);
9717
9718#if 0 /** @todo fix me */
9719 pDbgState->fClearCr0Mask = true;
9720 pDbgState->fClearCr4Mask = true;
9721#endif
9722 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9723 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9724 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9725 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9726 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9727 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9728 require clearing here and in the loop if we start using it. */
9729 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9730 }
9731 else
9732 {
9733 if (pDbgState->fClearCr0Mask)
9734 {
9735 pDbgState->fClearCr0Mask = false;
9736 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9737 }
9738 if (pDbgState->fClearCr4Mask)
9739 {
9740 pDbgState->fClearCr4Mask = false;
9741 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9742 }
9743 }
9744 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9745 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9746
9747 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9748 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9749 {
9750 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9751 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9752 }
9753 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9754 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9755
9756 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9757 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9758 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9759 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9760 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9761 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9762 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9763 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9764#if 0 /** @todo too slow, fix handler. */
9765 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9766#endif
9767 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9768
9769 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9770 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9771 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9772 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9773 {
9774 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9775 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9776 }
9777 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9778 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9779 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9780 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9781
9782 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9783 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9784 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9785 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9786 {
9787 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9788 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9789 }
9790 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9791 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9792 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9793 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9794
9795 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9796 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9797 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9798 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9799 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9800 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9801 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9802 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9803 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9804 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9805 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9806 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9807 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9808 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9809 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9810 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9811 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9812 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9813 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9814 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9815 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9816 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9817
9818#undef IS_EITHER_ENABLED
9819#undef SET_ONLY_XBM_IF_EITHER_EN
9820#undef SET_CPE1_XBM_IF_EITHER_EN
9821#undef SET_CPEU_XBM_IF_EITHER_EN
9822#undef SET_CPE2_XBM_IF_EITHER_EN
9823
9824 /*
9825 * Sanitize the control stuff.
9826 */
9827 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9828 if (pDbgState->fCpe2Extra)
9829 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9830 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9831 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9832 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9833 {
9834 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9835 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9836 }
9837
9838 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9839 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9840 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9841 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9842}
9843
9844
9845/**
9846 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9847 * appropriate.
9848 *
9849 * The caller has checked the VM-exit against the
9850 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9851 * already, so we don't have to do that either.
9852 *
9853 * @returns Strict VBox status code (i.e. informational status codes too).
9854 * @param pVM The cross context VM structure.
9855 * @param pVCpu The cross context virtual CPU structure.
9856 * @param pMixedCtx Pointer to the guest-CPU context.
9857 * @param pVmxTransient Pointer to the VMX-transient structure.
9858 * @param uExitReason The VM-exit reason.
9859 *
9860 * @remarks The name of this function is displayed by dtrace, so keep it short
9861 * and to the point. No longer than 33 chars long, please.
9862 */
9863static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9864 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9865{
9866 /*
9867 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9868 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9869 *
9870 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9871 * does. Must add/change/remove both places. Same ordering, please.
9872 *
9873 * Added/removed events must also be reflected in the next section
9874 * where we dispatch dtrace events.
9875 */
9876 bool fDtrace1 = false;
9877 bool fDtrace2 = false;
9878 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9879 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9880 uint32_t uEventArg = 0;
9881#define SET_EXIT(a_EventSubName) \
9882 do { \
9883 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9884 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9885 } while (0)
9886#define SET_BOTH(a_EventSubName) \
9887 do { \
9888 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9889 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9890 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9891 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9892 } while (0)
9893 switch (uExitReason)
9894 {
9895 case VMX_EXIT_MTF:
9896 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9897
9898 case VMX_EXIT_XCPT_OR_NMI:
9899 {
9900 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9901 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9902 {
9903 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9904 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9905 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9906 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9907 {
9908 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9909 {
9910 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9911 uEventArg = pVmxTransient->uExitIntErrorCode;
9912 }
9913 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9914 switch (enmEvent1)
9915 {
9916 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9917 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9918 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9919 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9920 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9921 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9922 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9923 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9924 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9925 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9926 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9927 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9928 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9929 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9930 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9931 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9932 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9933 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9934 default: break;
9935 }
9936 }
9937 else
9938 AssertFailed();
9939 break;
9940
9941 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9942 uEventArg = idxVector;
9943 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9944 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9945 break;
9946 }
9947 break;
9948 }
9949
9950 case VMX_EXIT_TRIPLE_FAULT:
9951 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9952 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9953 break;
9954 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9955 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9956 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9957 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9958 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9959
9960 /* Instruction specific VM-exits: */
9961 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9962 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9963 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9964 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9965 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9966 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9967 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9968 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9969 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9970 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9971 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9972 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9973 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9974 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9975 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9976 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9977 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9978 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9979 case VMX_EXIT_MOV_CRX:
9980 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9981/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9982* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9983 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9984 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9985 SET_BOTH(CRX_READ);
9986 else
9987 SET_BOTH(CRX_WRITE);
9988 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9989 break;
9990 case VMX_EXIT_MOV_DRX:
9991 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9992 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9993 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9994 SET_BOTH(DRX_READ);
9995 else
9996 SET_BOTH(DRX_WRITE);
9997 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9998 break;
9999 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10000 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10001 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10002 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10003 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10004 case VMX_EXIT_XDTR_ACCESS:
10005 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10006 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10007 {
10008 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10009 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10010 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10011 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10012 }
10013 break;
10014
10015 case VMX_EXIT_TR_ACCESS:
10016 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10017 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10018 {
10019 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10020 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10021 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10022 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10023 }
10024 break;
10025
10026 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10027 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10028 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10029 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10030 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10031 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10032 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10033 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10034 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10035 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10036 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10037
10038 /* Events that aren't relevant at this point. */
10039 case VMX_EXIT_EXT_INT:
10040 case VMX_EXIT_INT_WINDOW:
10041 case VMX_EXIT_NMI_WINDOW:
10042 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10043 case VMX_EXIT_PREEMPT_TIMER:
10044 case VMX_EXIT_IO_INSTR:
10045 break;
10046
10047 /* Errors and unexpected events. */
10048 case VMX_EXIT_INIT_SIGNAL:
10049 case VMX_EXIT_SIPI:
10050 case VMX_EXIT_IO_SMI:
10051 case VMX_EXIT_SMI:
10052 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10053 case VMX_EXIT_ERR_MSR_LOAD:
10054 case VMX_EXIT_ERR_MACHINE_CHECK:
10055 break;
10056
10057 default:
10058 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10059 break;
10060 }
10061#undef SET_BOTH
10062#undef SET_EXIT
10063
10064 /*
10065 * Dtrace tracepoints go first. We do them here at once so we don't
10066 * have to copy the guest state saving and stuff a few dozen times.
10067 * Down side is that we've got to repeat the switch, though this time
10068 * we use enmEvent since the probes are a subset of what DBGF does.
10069 */
10070 if (fDtrace1 || fDtrace2)
10071 {
10072 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10073 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10074 switch (enmEvent1)
10075 {
10076 /** @todo consider which extra parameters would be helpful for each probe. */
10077 case DBGFEVENT_END: break;
10078 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10079 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10080 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10081 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10082 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10083 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10084 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10085 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10086 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10087 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10088 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10089 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10090 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10091 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10092 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10093 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10094 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10095 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10096 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10097 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10098 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10099 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10100 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10101 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10102 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10103 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10104 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10105 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10106 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10107 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10108 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10109 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10110 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10111 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10112 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10113 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10114 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10115 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10116 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10117 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10118 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10119 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10120 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10121 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10122 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10123 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10124 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10125 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10126 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10127 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10128 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10129 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10130 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10131 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10132 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10133 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10134 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10135 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10136 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10137 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10138 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10139 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10140 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10141 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10142 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10143 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10144 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10145 }
10146 switch (enmEvent2)
10147 {
10148 /** @todo consider which extra parameters would be helpful for each probe. */
10149 case DBGFEVENT_END: break;
10150 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10151 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10152 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10153 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10154 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10155 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10156 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10157 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10158 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10159 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10160 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10161 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10162 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10163 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10164 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10165 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10166 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10167 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10168 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10169 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10170 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10171 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10172 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10173 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10178 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10179 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10180 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10181 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10182 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10183 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10184 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10185 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10186 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10187 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10188 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10189 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10190 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10191 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10192 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10194 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10197 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10198 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10199 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10200 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10201 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10202 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10203 }
10204 }
10205
10206 /*
10207 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10208 * the DBGF call will do a full check).
10209 *
10210 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10211 * Note! If we have to events, we prioritize the first, i.e. the instruction
10212 * one, in order to avoid event nesting.
10213 */
10214 if ( enmEvent1 != DBGFEVENT_END
10215 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10216 {
10217 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10218 if (rcStrict != VINF_SUCCESS)
10219 return rcStrict;
10220 }
10221 else if ( enmEvent2 != DBGFEVENT_END
10222 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10223 {
10224 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10225 if (rcStrict != VINF_SUCCESS)
10226 return rcStrict;
10227 }
10228
10229 return VINF_SUCCESS;
10230}
10231
10232
10233/**
10234 * Single-stepping VM-exit filtering.
10235 *
10236 * This is preprocessing the VM-exits and deciding whether we've gotten far
10237 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10238 * handling is performed.
10239 *
10240 * @returns Strict VBox status code (i.e. informational status codes too).
10241 * @param pVM The cross context VM structure.
10242 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10244 * out-of-sync. Make sure to update the required
10245 * fields before using them.
10246 * @param pVmxTransient Pointer to the VMX-transient structure.
10247 * @param uExitReason The VM-exit reason.
10248 * @param pDbgState The debug state.
10249 */
10250DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10251 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10252{
10253 /*
10254 * Expensive (saves context) generic dtrace VM-exit probe.
10255 */
10256 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10257 { /* more likely */ }
10258 else
10259 {
10260 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10261 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10262 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10263 }
10264
10265 /*
10266 * Check for host NMI, just to get that out of the way.
10267 */
10268 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10269 { /* normally likely */ }
10270 else
10271 {
10272 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10273 AssertRCReturn(rc2, rc2);
10274 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10275 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10276 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10277 }
10278
10279 /*
10280 * Check for single stepping event if we're stepping.
10281 */
10282 if (pVCpu->hm.s.fSingleInstruction)
10283 {
10284 switch (uExitReason)
10285 {
10286 case VMX_EXIT_MTF:
10287 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10288
10289 /* Various events: */
10290 case VMX_EXIT_XCPT_OR_NMI:
10291 case VMX_EXIT_EXT_INT:
10292 case VMX_EXIT_TRIPLE_FAULT:
10293 case VMX_EXIT_INT_WINDOW:
10294 case VMX_EXIT_NMI_WINDOW:
10295 case VMX_EXIT_TASK_SWITCH:
10296 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10297 case VMX_EXIT_APIC_ACCESS:
10298 case VMX_EXIT_EPT_VIOLATION:
10299 case VMX_EXIT_EPT_MISCONFIG:
10300 case VMX_EXIT_PREEMPT_TIMER:
10301
10302 /* Instruction specific VM-exits: */
10303 case VMX_EXIT_CPUID:
10304 case VMX_EXIT_GETSEC:
10305 case VMX_EXIT_HLT:
10306 case VMX_EXIT_INVD:
10307 case VMX_EXIT_INVLPG:
10308 case VMX_EXIT_RDPMC:
10309 case VMX_EXIT_RDTSC:
10310 case VMX_EXIT_RSM:
10311 case VMX_EXIT_VMCALL:
10312 case VMX_EXIT_VMCLEAR:
10313 case VMX_EXIT_VMLAUNCH:
10314 case VMX_EXIT_VMPTRLD:
10315 case VMX_EXIT_VMPTRST:
10316 case VMX_EXIT_VMREAD:
10317 case VMX_EXIT_VMRESUME:
10318 case VMX_EXIT_VMWRITE:
10319 case VMX_EXIT_VMXOFF:
10320 case VMX_EXIT_VMXON:
10321 case VMX_EXIT_MOV_CRX:
10322 case VMX_EXIT_MOV_DRX:
10323 case VMX_EXIT_IO_INSTR:
10324 case VMX_EXIT_RDMSR:
10325 case VMX_EXIT_WRMSR:
10326 case VMX_EXIT_MWAIT:
10327 case VMX_EXIT_MONITOR:
10328 case VMX_EXIT_PAUSE:
10329 case VMX_EXIT_XDTR_ACCESS:
10330 case VMX_EXIT_TR_ACCESS:
10331 case VMX_EXIT_INVEPT:
10332 case VMX_EXIT_RDTSCP:
10333 case VMX_EXIT_INVVPID:
10334 case VMX_EXIT_WBINVD:
10335 case VMX_EXIT_XSETBV:
10336 case VMX_EXIT_RDRAND:
10337 case VMX_EXIT_INVPCID:
10338 case VMX_EXIT_VMFUNC:
10339 case VMX_EXIT_RDSEED:
10340 case VMX_EXIT_XSAVES:
10341 case VMX_EXIT_XRSTORS:
10342 {
10343 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10344 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10345 AssertRCReturn(rc2, rc2);
10346 if ( pMixedCtx->rip != pDbgState->uRipStart
10347 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10348 return VINF_EM_DBG_STEPPED;
10349 break;
10350 }
10351
10352 /* Errors and unexpected events: */
10353 case VMX_EXIT_INIT_SIGNAL:
10354 case VMX_EXIT_SIPI:
10355 case VMX_EXIT_IO_SMI:
10356 case VMX_EXIT_SMI:
10357 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10358 case VMX_EXIT_ERR_MSR_LOAD:
10359 case VMX_EXIT_ERR_MACHINE_CHECK:
10360 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10361 break;
10362
10363 default:
10364 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10365 break;
10366 }
10367 }
10368
10369 /*
10370 * Check for debugger event breakpoints and dtrace probes.
10371 */
10372 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10373 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10374 {
10375 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10376 if (rcStrict != VINF_SUCCESS)
10377 return rcStrict;
10378 }
10379
10380 /*
10381 * Normal processing.
10382 */
10383#ifdef HMVMX_USE_FUNCTION_TABLE
10384 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10385#else
10386 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10387#endif
10388}
10389
10390
10391/**
10392 * Single steps guest code using VT-x.
10393 *
10394 * @returns Strict VBox status code (i.e. informational status codes too).
10395 * @param pVM The cross context VM structure.
10396 * @param pVCpu The cross context virtual CPU structure.
10397 * @param pCtx Pointer to the guest-CPU context.
10398 *
10399 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10400 */
10401static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10402{
10403 VMXTRANSIENT VmxTransient;
10404 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10405
10406 /* Set HMCPU indicators. */
10407 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10408 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10409 pVCpu->hm.s.fDebugWantRdTscExit = false;
10410 pVCpu->hm.s.fUsingDebugLoop = true;
10411
10412 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10413 VMXRUNDBGSTATE DbgState;
10414 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10415 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10416
10417 /*
10418 * The loop.
10419 */
10420 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10421 for (uint32_t cLoops = 0; ; cLoops++)
10422 {
10423 Assert(!HMR0SuspendPending());
10424 HMVMX_ASSERT_CPU_SAFE();
10425 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10426
10427 /*
10428 * Preparatory work for running guest code, this may force us to return
10429 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10430 */
10431 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10432 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10433 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10434 if (rcStrict != VINF_SUCCESS)
10435 break;
10436
10437 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10438 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10439
10440 /*
10441 * Now we can run the guest code.
10442 */
10443 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10444
10445 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10446
10447 /*
10448 * Restore any residual host-state and save any bits shared between host
10449 * and guest into the guest-CPU state. Re-enables interrupts!
10450 */
10451 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10452
10453 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10454 if (RT_SUCCESS(rcRun))
10455 { /* very likely */ }
10456 else
10457 {
10458 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10459 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10460 return rcRun;
10461 }
10462
10463 /* Profile the VM-exit. */
10464 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10466 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10467 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10468 HMVMX_START_EXIT_DISPATCH_PROF();
10469
10470 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10471
10472 /*
10473 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10474 */
10475 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10476 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10477 if (rcStrict != VINF_SUCCESS)
10478 break;
10479 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10480 {
10481 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10482 rcStrict = VINF_EM_RAW_INTERRUPT;
10483 break;
10484 }
10485
10486 /*
10487 * Stepping: Did the RIP change, if so, consider it a single step.
10488 * Otherwise, make sure one of the TFs gets set.
10489 */
10490 if (fStepping)
10491 {
10492 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10493 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10494 AssertRCReturn(rc2, rc2);
10495 if ( pCtx->rip != DbgState.uRipStart
10496 || pCtx->cs.Sel != DbgState.uCsStart)
10497 {
10498 rcStrict = VINF_EM_DBG_STEPPED;
10499 break;
10500 }
10501 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10502 }
10503
10504 /*
10505 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10506 */
10507 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10508 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10509 }
10510
10511 /*
10512 * Clear the X86_EFL_TF if necessary.
10513 */
10514 if (pVCpu->hm.s.fClearTrapFlag)
10515 {
10516 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10517 AssertRCReturn(rc2, rc2);
10518 pVCpu->hm.s.fClearTrapFlag = false;
10519 pCtx->eflags.Bits.u1TF = 0;
10520 }
10521 /** @todo there seems to be issues with the resume flag when the monitor trap
10522 * flag is pending without being used. Seen early in bios init when
10523 * accessing APIC page in protected mode. */
10524
10525 /*
10526 * Restore VM-exit control settings as we may not reenter this function the
10527 * next time around.
10528 */
10529 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10530
10531 /* Restore HMCPU indicators. */
10532 pVCpu->hm.s.fUsingDebugLoop = false;
10533 pVCpu->hm.s.fDebugWantRdTscExit = false;
10534 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10535
10536 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10537 return rcStrict;
10538}
10539
10540
10541/** @} */
10542
10543
10544/**
10545 * Checks if any expensive dtrace probes are enabled and we should go to the
10546 * debug loop.
10547 *
10548 * @returns true if we should use debug loop, false if not.
10549 */
10550static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10551{
10552 /* It's probably faster to OR the raw 32-bit counter variables together.
10553 Since the variables are in an array and the probes are next to one
10554 another (more or less), we have good locality. So, better read
10555 eight-nine cache lines ever time and only have one conditional, than
10556 128+ conditionals, right? */
10557 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10558 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10559 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10560 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10561 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10562 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10563 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10564 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10565 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10566 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10567 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10568 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10569 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10570 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10571 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10572 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10573 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10574 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10575 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10576 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10577 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10578 ) != 0
10579 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10580 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10581 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10582 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10583 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10584 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10585 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10586 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10587 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10588 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10589 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10590 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10591 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10592 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10593 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10594 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10595 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10596 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10597 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10598 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10599 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10600 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10601 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10602 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10603 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10604 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10605 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10606 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10607 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10608 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10609 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10610 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10611 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10612 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10613 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10614 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10615 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10616 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10617 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10618 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10619 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10620 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10621 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10622 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10623 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10624 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10625 ) != 0
10626 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10627 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10628 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10629 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10630 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10631 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10632 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10633 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10634 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10635 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10636 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10637 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10638 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10639 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10640 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10641 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10642 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10643 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10644 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10645 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10646 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10647 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10648 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10649 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10650 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10651 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10652 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10653 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10654 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10655 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10656 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10657 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10658 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10659 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10660 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10661 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10662 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10663 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10664 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10665 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10666 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10667 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10668 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10669 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10670 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10671 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10672 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10673 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10674 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10675 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10676 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10677 ) != 0;
10678}
10679
10680
10681/**
10682 * Runs the guest code using VT-x.
10683 *
10684 * @returns Strict VBox status code (i.e. informational status codes too).
10685 * @param pVM The cross context VM structure.
10686 * @param pVCpu The cross context virtual CPU structure.
10687 * @param pCtx Pointer to the guest-CPU context.
10688 */
10689VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10690{
10691 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10692 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10693 HMVMX_ASSERT_PREEMPT_SAFE();
10694
10695 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10696
10697 VBOXSTRICTRC rcStrict;
10698 if ( !pVCpu->hm.s.fUseDebugLoop
10699 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10700 && !DBGFIsStepping(pVCpu)
10701 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10702 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10703 else
10704 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10705
10706 if (rcStrict == VERR_EM_INTERPRETER)
10707 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10708 else if (rcStrict == VINF_EM_RESET)
10709 rcStrict = VINF_EM_TRIPLE_FAULT;
10710
10711 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10712 if (RT_FAILURE(rc2))
10713 {
10714 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10715 rcStrict = rc2;
10716 }
10717 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10718 return rcStrict;
10719}
10720
10721
10722#ifndef HMVMX_USE_FUNCTION_TABLE
10723DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10724{
10725# ifdef DEBUG_ramshankar
10726# define RETURN_EXIT_CALL(a_CallExpr) \
10727 do { \
10728 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10729 VBOXSTRICTRC rcStrict = a_CallExpr; \
10730 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10731 return rcStrict; \
10732 } while (0)
10733# else
10734# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10735# endif
10736 switch (rcReason)
10737 {
10738 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10739 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10740 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10741 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10742 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10743 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10744 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10745 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10746 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10747 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10748 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10749 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10750 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10751 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10752 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10753 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10754 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10755 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10756 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10757 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10758 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10759 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10760 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10761 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10762 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10763 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10764 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10765 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10766 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10767 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10768 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10769 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10770 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10771 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10772
10773 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10774 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10775 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10776 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10777 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10778 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10779 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10780 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10781 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10782
10783 case VMX_EXIT_VMCLEAR:
10784 case VMX_EXIT_VMLAUNCH:
10785 case VMX_EXIT_VMPTRLD:
10786 case VMX_EXIT_VMPTRST:
10787 case VMX_EXIT_VMREAD:
10788 case VMX_EXIT_VMRESUME:
10789 case VMX_EXIT_VMWRITE:
10790 case VMX_EXIT_VMXOFF:
10791 case VMX_EXIT_VMXON:
10792 case VMX_EXIT_INVEPT:
10793 case VMX_EXIT_INVVPID:
10794 case VMX_EXIT_VMFUNC:
10795 case VMX_EXIT_XSAVES:
10796 case VMX_EXIT_XRSTORS:
10797 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10798 case VMX_EXIT_ENCLS:
10799 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10800 case VMX_EXIT_PML_FULL:
10801 default:
10802 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10803 }
10804#undef RETURN_EXIT_CALL
10805}
10806#endif /* !HMVMX_USE_FUNCTION_TABLE */
10807
10808
10809#ifdef VBOX_STRICT
10810/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10811# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10812 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10813
10814# define HMVMX_ASSERT_PREEMPT_CPUID() \
10815 do { \
10816 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10817 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10818 } while (0)
10819
10820# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10821 do { \
10822 AssertPtr(pVCpu); \
10823 AssertPtr(pMixedCtx); \
10824 AssertPtr(pVmxTransient); \
10825 Assert(pVmxTransient->fVMEntryFailed == false); \
10826 Assert(ASMIntAreEnabled()); \
10827 HMVMX_ASSERT_PREEMPT_SAFE(); \
10828 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10829 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10830 HMVMX_ASSERT_PREEMPT_SAFE(); \
10831 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10832 HMVMX_ASSERT_PREEMPT_CPUID(); \
10833 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10834 } while (0)
10835
10836# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10837 do { \
10838 Log4Func(("\n")); \
10839 } while (0)
10840#else /* nonstrict builds: */
10841# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10842 do { \
10843 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10844 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10845 } while (0)
10846# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10847#endif
10848
10849
10850/**
10851 * Advances the guest RIP by the specified number of bytes.
10852 *
10853 * @param pVCpu The cross context virtual CPU structure.
10854 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10855 * out-of-sync. Make sure to update the required fields
10856 * before using them.
10857 * @param cbInstr Number of bytes to advance the RIP by.
10858 *
10859 * @remarks No-long-jump zone!!!
10860 */
10861DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10862{
10863 /* Advance the RIP. */
10864 pMixedCtx->rip += cbInstr;
10865 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10866
10867 /* Update interrupt inhibition. */
10868 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10869 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10870 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10871}
10872
10873
10874/**
10875 * Advances the guest RIP after reading it from the VMCS.
10876 *
10877 * @returns VBox status code, no informational status codes.
10878 * @param pVCpu The cross context virtual CPU structure.
10879 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10880 * out-of-sync. Make sure to update the required fields
10881 * before using them.
10882 * @param pVmxTransient Pointer to the VMX transient structure.
10883 *
10884 * @remarks No-long-jump zone!!!
10885 */
10886static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10887{
10888 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10889 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10890 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10891 AssertRCReturn(rc, rc);
10892
10893 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10894
10895 /*
10896 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10897 * pending debug exception field as it takes care of priority of events.
10898 *
10899 * See Intel spec. 32.2.1 "Debug Exceptions".
10900 */
10901 if ( !pVCpu->hm.s.fSingleInstruction
10902 && pMixedCtx->eflags.Bits.u1TF)
10903 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10904
10905 return VINF_SUCCESS;
10906}
10907
10908
10909/**
10910 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10911 * and update error record fields accordingly.
10912 *
10913 * @return VMX_IGS_* return codes.
10914 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10915 * wrong with the guest state.
10916 *
10917 * @param pVM The cross context VM structure.
10918 * @param pVCpu The cross context virtual CPU structure.
10919 * @param pCtx Pointer to the guest-CPU state.
10920 *
10921 * @remarks This function assumes our cache of the VMCS controls
10922 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10923 */
10924static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10925{
10926#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10927#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10928 uError = (err); \
10929 break; \
10930 } else do { } while (0)
10931
10932 int rc;
10933 uint32_t uError = VMX_IGS_ERROR;
10934 uint32_t u32Val;
10935 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10936
10937 do
10938 {
10939 /*
10940 * CR0.
10941 */
10942 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10943 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10944 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10945 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10946 if (fUnrestrictedGuest)
10947 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10948
10949 uint32_t u32GuestCR0;
10950 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10951 AssertRCBreak(rc);
10952 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10953 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10954 if ( !fUnrestrictedGuest
10955 && (u32GuestCR0 & X86_CR0_PG)
10956 && !(u32GuestCR0 & X86_CR0_PE))
10957 {
10958 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10959 }
10960
10961 /*
10962 * CR4.
10963 */
10964 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10965 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10966
10967 uint32_t u32GuestCR4;
10968 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10969 AssertRCBreak(rc);
10970 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10971 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10972
10973 /*
10974 * IA32_DEBUGCTL MSR.
10975 */
10976 uint64_t u64Val;
10977 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10978 AssertRCBreak(rc);
10979 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10980 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10981 {
10982 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10983 }
10984 uint64_t u64DebugCtlMsr = u64Val;
10985
10986#ifdef VBOX_STRICT
10987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10988 AssertRCBreak(rc);
10989 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10990#endif
10991 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10992
10993 /*
10994 * RIP and RFLAGS.
10995 */
10996 uint32_t u32Eflags;
10997#if HC_ARCH_BITS == 64
10998 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10999 AssertRCBreak(rc);
11000 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11001 if ( !fLongModeGuest
11002 || !pCtx->cs.Attr.n.u1Long)
11003 {
11004 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11005 }
11006 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11007 * must be identical if the "IA-32e mode guest" VM-entry
11008 * control is 1 and CS.L is 1. No check applies if the
11009 * CPU supports 64 linear-address bits. */
11010
11011 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11012 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11013 AssertRCBreak(rc);
11014 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11015 VMX_IGS_RFLAGS_RESERVED);
11016 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11017 u32Eflags = u64Val;
11018#else
11019 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11020 AssertRCBreak(rc);
11021 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11022 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11023#endif
11024
11025 if ( fLongModeGuest
11026 || ( fUnrestrictedGuest
11027 && !(u32GuestCR0 & X86_CR0_PE)))
11028 {
11029 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11030 }
11031
11032 uint32_t u32EntryInfo;
11033 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11034 AssertRCBreak(rc);
11035 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11036 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11037 {
11038 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11039 }
11040
11041 /*
11042 * 64-bit checks.
11043 */
11044#if HC_ARCH_BITS == 64
11045 if (fLongModeGuest)
11046 {
11047 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11048 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11049 }
11050
11051 if ( !fLongModeGuest
11052 && (u32GuestCR4 & X86_CR4_PCIDE))
11053 {
11054 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11055 }
11056
11057 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11058 * 51:32 beyond the processor's physical-address width are 0. */
11059
11060 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11061 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11062 {
11063 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11064 }
11065
11066 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11067 AssertRCBreak(rc);
11068 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11069
11070 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11071 AssertRCBreak(rc);
11072 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11073#endif
11074
11075 /*
11076 * PERF_GLOBAL MSR.
11077 */
11078 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11079 {
11080 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11081 AssertRCBreak(rc);
11082 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11083 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11084 }
11085
11086 /*
11087 * PAT MSR.
11088 */
11089 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11090 {
11091 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11092 AssertRCBreak(rc);
11093 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11094 for (unsigned i = 0; i < 8; i++)
11095 {
11096 uint8_t u8Val = (u64Val & 0xff);
11097 if ( u8Val != 0 /* UC */
11098 && u8Val != 1 /* WC */
11099 && u8Val != 4 /* WT */
11100 && u8Val != 5 /* WP */
11101 && u8Val != 6 /* WB */
11102 && u8Val != 7 /* UC- */)
11103 {
11104 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11105 }
11106 u64Val >>= 8;
11107 }
11108 }
11109
11110 /*
11111 * EFER MSR.
11112 */
11113 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11114 {
11115 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11116 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11117 AssertRCBreak(rc);
11118 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11119 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11120 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11121 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11122 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11123 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11124 || !(u32GuestCR0 & X86_CR0_PG)
11125 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11126 VMX_IGS_EFER_LMA_LME_MISMATCH);
11127 }
11128
11129 /*
11130 * Segment registers.
11131 */
11132 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11133 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11134 if (!(u32Eflags & X86_EFL_VM))
11135 {
11136 /* CS */
11137 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11138 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11139 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11140 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11141 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11142 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11143 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11144 /* CS cannot be loaded with NULL in protected mode. */
11145 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11146 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11147 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11148 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11149 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11150 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11151 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11152 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11153 else
11154 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11155
11156 /* SS */
11157 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11158 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11159 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11160 if ( !(pCtx->cr0 & X86_CR0_PE)
11161 || pCtx->cs.Attr.n.u4Type == 3)
11162 {
11163 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11164 }
11165 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11166 {
11167 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11168 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11169 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11170 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11171 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11172 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11173 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11174 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11175 }
11176
11177 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11178 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11179 {
11180 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11181 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11182 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11183 || pCtx->ds.Attr.n.u4Type > 11
11184 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11185 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11186 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11187 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11188 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11189 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11190 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11191 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11192 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11193 }
11194 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11195 {
11196 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11197 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11198 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11199 || pCtx->es.Attr.n.u4Type > 11
11200 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11201 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11202 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11203 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11204 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11205 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11206 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11207 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11208 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11209 }
11210 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11211 {
11212 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11213 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11214 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11215 || pCtx->fs.Attr.n.u4Type > 11
11216 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11217 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11218 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11219 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11220 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11221 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11222 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11223 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11224 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11225 }
11226 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11227 {
11228 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11229 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11230 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11231 || pCtx->gs.Attr.n.u4Type > 11
11232 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11233 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11234 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11235 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11236 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11237 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11238 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11239 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11240 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11241 }
11242 /* 64-bit capable CPUs. */
11243#if HC_ARCH_BITS == 64
11244 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11245 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11246 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11247 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11248 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11249 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11250 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11251 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11252 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11253 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11254 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11255#endif
11256 }
11257 else
11258 {
11259 /* V86 mode checks. */
11260 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11261 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11262 {
11263 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11264 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11265 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11266 }
11267 else
11268 {
11269 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11270 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11271 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11272 }
11273
11274 /* CS */
11275 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11276 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11277 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11278 /* SS */
11279 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11280 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11281 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11282 /* DS */
11283 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11284 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11285 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11286 /* ES */
11287 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11288 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11289 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11290 /* FS */
11291 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11292 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11293 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11294 /* GS */
11295 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11296 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11297 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11298 /* 64-bit capable CPUs. */
11299#if HC_ARCH_BITS == 64
11300 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11301 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11302 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11303 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11304 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11305 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11306 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11307 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11308 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11309 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11310 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11311#endif
11312 }
11313
11314 /*
11315 * TR.
11316 */
11317 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11318 /* 64-bit capable CPUs. */
11319#if HC_ARCH_BITS == 64
11320 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11321#endif
11322 if (fLongModeGuest)
11323 {
11324 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11325 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11326 }
11327 else
11328 {
11329 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11330 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11331 VMX_IGS_TR_ATTR_TYPE_INVALID);
11332 }
11333 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11334 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11335 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11336 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11337 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11338 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11339 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11340 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11341
11342 /*
11343 * GDTR and IDTR.
11344 */
11345#if HC_ARCH_BITS == 64
11346 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11347 AssertRCBreak(rc);
11348 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11349
11350 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11351 AssertRCBreak(rc);
11352 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11353#endif
11354
11355 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11356 AssertRCBreak(rc);
11357 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11358
11359 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11360 AssertRCBreak(rc);
11361 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11362
11363 /*
11364 * Guest Non-Register State.
11365 */
11366 /* Activity State. */
11367 uint32_t u32ActivityState;
11368 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11369 AssertRCBreak(rc);
11370 HMVMX_CHECK_BREAK( !u32ActivityState
11371 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11372 VMX_IGS_ACTIVITY_STATE_INVALID);
11373 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11374 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11375 uint32_t u32IntrState;
11376 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11377 AssertRCBreak(rc);
11378 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11379 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11380 {
11381 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11382 }
11383
11384 /** @todo Activity state and injecting interrupts. Left as a todo since we
11385 * currently don't use activity states but ACTIVE. */
11386
11387 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11388 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11389
11390 /* Guest interruptibility-state. */
11391 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11392 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11393 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11394 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11395 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11396 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11397 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11398 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11399 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11400 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11401 {
11402 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11403 {
11404 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11405 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11406 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11407 }
11408 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11409 {
11410 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11411 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11412 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11413 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11414 }
11415 }
11416 /** @todo Assumes the processor is not in SMM. */
11417 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11418 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11419 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11420 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11421 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11422 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11423 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11424 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11425 {
11426 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11427 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11428 }
11429
11430 /* Pending debug exceptions. */
11431#if HC_ARCH_BITS == 64
11432 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11433 AssertRCBreak(rc);
11434 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11435 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11436 u32Val = u64Val; /* For pending debug exceptions checks below. */
11437#else
11438 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11439 AssertRCBreak(rc);
11440 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11441 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11442#endif
11443
11444 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11445 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11446 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11447 {
11448 if ( (u32Eflags & X86_EFL_TF)
11449 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11450 {
11451 /* Bit 14 is PendingDebug.BS. */
11452 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11453 }
11454 if ( !(u32Eflags & X86_EFL_TF)
11455 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11456 {
11457 /* Bit 14 is PendingDebug.BS. */
11458 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11459 }
11460 }
11461
11462 /* VMCS link pointer. */
11463 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11464 AssertRCBreak(rc);
11465 if (u64Val != UINT64_C(0xffffffffffffffff))
11466 {
11467 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11468 /** @todo Bits beyond the processor's physical-address width MBZ. */
11469 /** @todo 32-bit located in memory referenced by value of this field (as a
11470 * physical address) must contain the processor's VMCS revision ID. */
11471 /** @todo SMM checks. */
11472 }
11473
11474 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11475 * not using Nested Paging? */
11476 if ( pVM->hm.s.fNestedPaging
11477 && !fLongModeGuest
11478 && CPUMIsGuestInPAEModeEx(pCtx))
11479 {
11480 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11481 AssertRCBreak(rc);
11482 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11483
11484 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11485 AssertRCBreak(rc);
11486 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11487
11488 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11489 AssertRCBreak(rc);
11490 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11491
11492 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11493 AssertRCBreak(rc);
11494 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11495 }
11496
11497 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11498 if (uError == VMX_IGS_ERROR)
11499 uError = VMX_IGS_REASON_NOT_FOUND;
11500 } while (0);
11501
11502 pVCpu->hm.s.u32HMError = uError;
11503 return uError;
11504
11505#undef HMVMX_ERROR_BREAK
11506#undef HMVMX_CHECK_BREAK
11507}
11508
11509/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11510/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11511/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11512
11513/** @name VM-exit handlers.
11514 * @{
11515 */
11516
11517/**
11518 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11524 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11525 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11526 return VINF_SUCCESS;
11527 return VINF_EM_RAW_INTERRUPT;
11528}
11529
11530
11531/**
11532 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11533 */
11534HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11537 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11538
11539 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11540 AssertRCReturn(rc, rc);
11541
11542 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11543 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11544 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11545 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11546
11547 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11548 {
11549 /*
11550 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11551 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11552 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11553 *
11554 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11555 */
11556 VMXDispatchHostNmi();
11557 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11558 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11559 return VINF_SUCCESS;
11560 }
11561
11562 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11563 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11564 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11565 { /* likely */ }
11566 else
11567 {
11568 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11569 rcStrictRc1 = VINF_SUCCESS;
11570 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11571 return rcStrictRc1;
11572 }
11573
11574 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11575 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11576 switch (uIntType)
11577 {
11578 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11579 Assert(uVector == X86_XCPT_DB);
11580 /* fall thru */
11581 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11582 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11583 /* fall thru */
11584 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11585 {
11586 /*
11587 * If there's any exception caused as a result of event injection, go back to
11588 * the interpreter. The page-fault case is complicated and we manually handle
11589 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11590 * handled in hmR0VmxCheckExitDueToEventDelivery.
11591 */
11592 if (!pVCpu->hm.s.Event.fPending)
11593 { /* likely */ }
11594 else if ( uVector != X86_XCPT_PF
11595 && uVector != X86_XCPT_AC)
11596 {
11597 /** @todo Why do we need to fallback to the interpreter here? */
11598 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11599 rc = VERR_EM_INTERPRETER;
11600 break;
11601 }
11602
11603 switch (uVector)
11604 {
11605 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11606 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11607 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11608 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11609 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11610 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11611 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11612
11613 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11614 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11615 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11616 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11617 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11618 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11619 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11620 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11621 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11622 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11623 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11624 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11625 default:
11626 {
11627 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11628 AssertRCReturn(rc, rc);
11629
11630 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11631 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11632 {
11633 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11634 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11635 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11636
11637 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11638 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11639 AssertRCReturn(rc, rc);
11640 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11641 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11642 0 /* GCPtrFaultAddress */);
11643 AssertRCReturn(rc, rc);
11644 }
11645 else
11646 {
11647 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11648 pVCpu->hm.s.u32HMError = uVector;
11649 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11650 }
11651 break;
11652 }
11653 }
11654 break;
11655 }
11656
11657 default:
11658 {
11659 pVCpu->hm.s.u32HMError = uExitIntInfo;
11660 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11661 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11662 break;
11663 }
11664 }
11665 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11666 return rc;
11667}
11668
11669
11670/**
11671 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11672 */
11673HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11674{
11675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11676
11677 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11678 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11679
11680 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11681 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11682 return VINF_SUCCESS;
11683}
11684
11685
11686/**
11687 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11688 */
11689HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11690{
11691 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11692 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11693 {
11694 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11695 HMVMX_RETURN_UNEXPECTED_EXIT();
11696 }
11697
11698 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11699
11700 /*
11701 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11702 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11703 */
11704 uint32_t uIntrState = 0;
11705 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11706 AssertRCReturn(rc, rc);
11707
11708 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11709 if ( fBlockSti
11710 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11711 {
11712 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11713 }
11714
11715 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11716 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11717
11718 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11719 return VINF_SUCCESS;
11720}
11721
11722
11723/**
11724 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11725 */
11726HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11727{
11728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11729 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11730 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11731}
11732
11733
11734/**
11735 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11736 */
11737HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11738{
11739 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11740 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11741 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11742}
11743
11744
11745/**
11746 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11747 */
11748HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11749{
11750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11751 PVM pVM = pVCpu->CTX_SUFF(pVM);
11752 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11753 if (RT_LIKELY(rc == VINF_SUCCESS))
11754 {
11755 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11756 Assert(pVmxTransient->cbInstr == 2);
11757 }
11758 else
11759 {
11760 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11761 rc = VERR_EM_INTERPRETER;
11762 }
11763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11764 return rc;
11765}
11766
11767
11768/**
11769 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11770 */
11771HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11772{
11773 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11774 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11775 AssertRCReturn(rc, rc);
11776
11777 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11778 return VINF_EM_RAW_EMULATE_INSTR;
11779
11780 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11781 HMVMX_RETURN_UNEXPECTED_EXIT();
11782}
11783
11784
11785/**
11786 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11787 */
11788HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11789{
11790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11791 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11792 AssertRCReturn(rc, rc);
11793
11794 PVM pVM = pVCpu->CTX_SUFF(pVM);
11795 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11796 if (RT_LIKELY(rc == VINF_SUCCESS))
11797 {
11798 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11799 Assert(pVmxTransient->cbInstr == 2);
11800 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11801 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11802 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11803 }
11804 else
11805 rc = VERR_EM_INTERPRETER;
11806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11807 return rc;
11808}
11809
11810
11811/**
11812 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11813 */
11814HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11815{
11816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11817 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11818 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11819 AssertRCReturn(rc, rc);
11820
11821 PVM pVM = pVCpu->CTX_SUFF(pVM);
11822 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11823 if (RT_SUCCESS(rc))
11824 {
11825 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11826 Assert(pVmxTransient->cbInstr == 3);
11827 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11828 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11829 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11830 }
11831 else
11832 {
11833 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11834 rc = VERR_EM_INTERPRETER;
11835 }
11836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11837 return rc;
11838}
11839
11840
11841/**
11842 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11843 */
11844HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11845{
11846 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11847 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11848 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11849 AssertRCReturn(rc, rc);
11850
11851 PVM pVM = pVCpu->CTX_SUFF(pVM);
11852 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11853 if (RT_LIKELY(rc == VINF_SUCCESS))
11854 {
11855 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11856 Assert(pVmxTransient->cbInstr == 2);
11857 }
11858 else
11859 {
11860 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11861 rc = VERR_EM_INTERPRETER;
11862 }
11863 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11864 return rc;
11865}
11866
11867
11868/**
11869 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11870 */
11871HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11872{
11873 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11874 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11875
11876 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11877 if (pVCpu->hm.s.fHypercallsEnabled)
11878 {
11879#if 0
11880 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11881#else
11882 /* Aggressive state sync. for now. */
11883 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11884 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11885 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11886 AssertRCReturn(rc, rc);
11887#endif
11888
11889 /* Perform the hypercall. */
11890 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11891 if (rcStrict == VINF_SUCCESS)
11892 {
11893 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11894 AssertRCReturn(rc, rc);
11895 }
11896 else
11897 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11898 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11899 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11900
11901 /* If the hypercall changes anything other than guest's general-purpose registers,
11902 we would need to reload the guest changed bits here before VM-entry. */
11903 }
11904 else
11905 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11906
11907 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11908 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11909 {
11910 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11911 rcStrict = VINF_SUCCESS;
11912 }
11913
11914 return rcStrict;
11915}
11916
11917
11918/**
11919 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11920 */
11921HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11922{
11923 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11924 PVM pVM = pVCpu->CTX_SUFF(pVM);
11925 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11926
11927 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11928 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11929 AssertRCReturn(rc, rc);
11930
11931 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11932 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11933 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11934 else
11935 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11936 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11938 return rcStrict;
11939}
11940
11941
11942/**
11943 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11944 */
11945HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11946{
11947 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11948 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11949 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11950 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11951 AssertRCReturn(rc, rc);
11952
11953 PVM pVM = pVCpu->CTX_SUFF(pVM);
11954 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11955 if (RT_LIKELY(rc == VINF_SUCCESS))
11956 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11957 else
11958 {
11959 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11960 rc = VERR_EM_INTERPRETER;
11961 }
11962 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11963 return rc;
11964}
11965
11966
11967/**
11968 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11969 */
11970HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11971{
11972 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11973 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11974 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11975 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11976 AssertRCReturn(rc, rc);
11977
11978 PVM pVM = pVCpu->CTX_SUFF(pVM);
11979 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11980 rc = VBOXSTRICTRC_VAL(rc2);
11981 if (RT_LIKELY( rc == VINF_SUCCESS
11982 || rc == VINF_EM_HALT))
11983 {
11984 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11985 AssertRCReturn(rc3, rc3);
11986
11987 if ( rc == VINF_EM_HALT
11988 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11989 {
11990 rc = VINF_SUCCESS;
11991 }
11992 }
11993 else
11994 {
11995 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11996 rc = VERR_EM_INTERPRETER;
11997 }
11998 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11999 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12001 return rc;
12002}
12003
12004
12005/**
12006 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12007 */
12008HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12009{
12010 /*
12011 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12012 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12013 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12014 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12015 */
12016 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12017 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12018 HMVMX_RETURN_UNEXPECTED_EXIT();
12019}
12020
12021
12022/**
12023 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12024 */
12025HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12026{
12027 /*
12028 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12029 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12030 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12031 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12032 */
12033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12034 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12035 HMVMX_RETURN_UNEXPECTED_EXIT();
12036}
12037
12038
12039/**
12040 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12041 */
12042HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12043{
12044 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12045 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12046 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12047 HMVMX_RETURN_UNEXPECTED_EXIT();
12048}
12049
12050
12051/**
12052 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12053 */
12054HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12055{
12056 /*
12057 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12058 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12059 * See Intel spec. 25.3 "Other Causes of VM-exits".
12060 */
12061 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12062 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12063 HMVMX_RETURN_UNEXPECTED_EXIT();
12064}
12065
12066
12067/**
12068 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12069 * VM-exit.
12070 */
12071HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12072{
12073 /*
12074 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12075 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12076 *
12077 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12078 * See Intel spec. "23.8 Restrictions on VMX operation".
12079 */
12080 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12081 return VINF_SUCCESS;
12082}
12083
12084
12085/**
12086 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12087 * VM-exit.
12088 */
12089HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12090{
12091 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12092 return VINF_EM_RESET;
12093}
12094
12095
12096/**
12097 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12098 */
12099HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12100{
12101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12102 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12103
12104 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12105 AssertRCReturn(rc, rc);
12106
12107 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12108 rc = VINF_SUCCESS;
12109 else
12110 rc = VINF_EM_HALT;
12111
12112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12113 if (rc != VINF_SUCCESS)
12114 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12115 return rc;
12116}
12117
12118
12119/**
12120 * VM-exit handler for instructions that result in a \#UD exception delivered to
12121 * the guest.
12122 */
12123HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12124{
12125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12126 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12127 return VINF_SUCCESS;
12128}
12129
12130
12131/**
12132 * VM-exit handler for expiry of the VMX preemption timer.
12133 */
12134HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12135{
12136 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12137
12138 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12139 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12140
12141 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12142 PVM pVM = pVCpu->CTX_SUFF(pVM);
12143 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12145 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12146}
12147
12148
12149/**
12150 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12151 */
12152HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12153{
12154 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12155
12156 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12157 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12158 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12159 AssertRCReturn(rc, rc);
12160
12161 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12162 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12163
12164 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12165
12166 return rcStrict;
12167}
12168
12169
12170/**
12171 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12172 */
12173HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12174{
12175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12176
12177 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
12178 /** @todo implement EMInterpretInvpcid() */
12179 return VERR_EM_INTERPRETER;
12180}
12181
12182
12183/**
12184 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12185 * Error VM-exit.
12186 */
12187HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12188{
12189 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12190 AssertRCReturn(rc, rc);
12191
12192 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12193 AssertRCReturn(rc, rc);
12194
12195 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12196 NOREF(uInvalidReason);
12197
12198#ifdef VBOX_STRICT
12199 uint32_t uIntrState;
12200 RTHCUINTREG uHCReg;
12201 uint64_t u64Val;
12202 uint32_t u32Val;
12203
12204 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12205 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12206 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12207 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12208 AssertRCReturn(rc, rc);
12209
12210 Log4(("uInvalidReason %u\n", uInvalidReason));
12211 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12212 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12213 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12214 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12215
12216 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12217 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12218 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12219 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12220 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12221 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12222 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12223 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12224 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12225 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12226 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12227 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12228#else
12229 NOREF(pVmxTransient);
12230#endif
12231
12232 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12233 return VERR_VMX_INVALID_GUEST_STATE;
12234}
12235
12236
12237/**
12238 * VM-exit handler for VM-entry failure due to an MSR-load
12239 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12240 */
12241HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12242{
12243 NOREF(pVmxTransient);
12244 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12245 HMVMX_RETURN_UNEXPECTED_EXIT();
12246}
12247
12248
12249/**
12250 * VM-exit handler for VM-entry failure due to a machine-check event
12251 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12252 */
12253HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12254{
12255 NOREF(pVmxTransient);
12256 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12257 HMVMX_RETURN_UNEXPECTED_EXIT();
12258}
12259
12260
12261/**
12262 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12263 * theory.
12264 */
12265HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12266{
12267 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12268 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12269 return VERR_VMX_UNDEFINED_EXIT_CODE;
12270}
12271
12272
12273/**
12274 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12275 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12276 * Conditional VM-exit.
12277 */
12278HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12279{
12280 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12281
12282 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12283 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12284 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12285 return VERR_EM_INTERPRETER;
12286 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12287 HMVMX_RETURN_UNEXPECTED_EXIT();
12288}
12289
12290
12291/**
12292 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12293 */
12294HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12295{
12296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12297
12298 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12299 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12300 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12301 return VERR_EM_INTERPRETER;
12302 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12303 HMVMX_RETURN_UNEXPECTED_EXIT();
12304}
12305
12306
12307/**
12308 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12309 */
12310HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12311{
12312 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12313
12314 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12315 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12316 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12317 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12318 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12319 {
12320 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12321 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12322 }
12323 AssertRCReturn(rc, rc);
12324 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12325
12326#ifdef VBOX_STRICT
12327 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12328 {
12329 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12330 && pMixedCtx->ecx != MSR_K6_EFER)
12331 {
12332 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12333 pMixedCtx->ecx));
12334 HMVMX_RETURN_UNEXPECTED_EXIT();
12335 }
12336 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12337 {
12338 VMXMSREXITREAD enmRead;
12339 VMXMSREXITWRITE enmWrite;
12340 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12341 AssertRCReturn(rc2, rc2);
12342 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12343 {
12344 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12345 HMVMX_RETURN_UNEXPECTED_EXIT();
12346 }
12347 }
12348 }
12349#endif
12350
12351 PVM pVM = pVCpu->CTX_SUFF(pVM);
12352 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12353 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12354 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12356 if (RT_SUCCESS(rc))
12357 {
12358 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12359 Assert(pVmxTransient->cbInstr == 2);
12360 }
12361 return rc;
12362}
12363
12364
12365/**
12366 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12367 */
12368HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12369{
12370 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12371 PVM pVM = pVCpu->CTX_SUFF(pVM);
12372 int rc = VINF_SUCCESS;
12373
12374 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12375 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12376 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12377 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12378 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12379 {
12380 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12381 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12382 }
12383 AssertRCReturn(rc, rc);
12384 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12385
12386 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12387 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12389
12390 if (RT_SUCCESS(rc))
12391 {
12392 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12393
12394 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12395 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12396 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12397 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12398 {
12399 /*
12400 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12401 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12402 * EMInterpretWrmsr() changes it.
12403 */
12404 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12405 }
12406 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12407 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12408 else if (pMixedCtx->ecx == MSR_K6_EFER)
12409 {
12410 /*
12411 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12412 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12413 * the other bits as well, SCE and NXE. See @bugref{7368}.
12414 */
12415 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12416 }
12417
12418 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12419 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12420 {
12421 switch (pMixedCtx->ecx)
12422 {
12423 /*
12424 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12425 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12426 */
12427 case MSR_IA32_SYSENTER_CS:
12428 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12429 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12430 break;
12431 case MSR_IA32_SYSENTER_EIP:
12432 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12433 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12434 break;
12435 case MSR_IA32_SYSENTER_ESP:
12436 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12437 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12438 break;
12439 case MSR_K8_FS_BASE: /* fall thru */
12440 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12441 case MSR_K6_EFER: /* already handled above */ break;
12442 default:
12443 {
12444 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12445 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12446 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12447 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12448 break;
12449 }
12450 }
12451 }
12452#ifdef VBOX_STRICT
12453 else
12454 {
12455 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12456 switch (pMixedCtx->ecx)
12457 {
12458 case MSR_IA32_SYSENTER_CS:
12459 case MSR_IA32_SYSENTER_EIP:
12460 case MSR_IA32_SYSENTER_ESP:
12461 case MSR_K8_FS_BASE:
12462 case MSR_K8_GS_BASE:
12463 {
12464 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12465 HMVMX_RETURN_UNEXPECTED_EXIT();
12466 }
12467
12468 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12469 default:
12470 {
12471 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12472 {
12473 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12474 if (pMixedCtx->ecx != MSR_K6_EFER)
12475 {
12476 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12477 pMixedCtx->ecx));
12478 HMVMX_RETURN_UNEXPECTED_EXIT();
12479 }
12480 }
12481
12482 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12483 {
12484 VMXMSREXITREAD enmRead;
12485 VMXMSREXITWRITE enmWrite;
12486 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12487 AssertRCReturn(rc2, rc2);
12488 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12489 {
12490 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12491 HMVMX_RETURN_UNEXPECTED_EXIT();
12492 }
12493 }
12494 break;
12495 }
12496 }
12497 }
12498#endif /* VBOX_STRICT */
12499 }
12500 return rc;
12501}
12502
12503
12504/**
12505 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12506 */
12507HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12508{
12509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12510
12511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12512 return VINF_EM_RAW_INTERRUPT;
12513}
12514
12515
12516/**
12517 * VM-exit handler for when the TPR value is lowered below the specified
12518 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12519 */
12520HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12521{
12522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12523 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12524
12525 /*
12526 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12527 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12528 */
12529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12530 return VINF_SUCCESS;
12531}
12532
12533
12534/**
12535 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12536 * VM-exit.
12537 *
12538 * @retval VINF_SUCCESS when guest execution can continue.
12539 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12540 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12541 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12542 * interpreter.
12543 */
12544HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12545{
12546 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12547 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12548 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12549 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12550 AssertRCReturn(rc, rc);
12551
12552 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12553 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12554 PVM pVM = pVCpu->CTX_SUFF(pVM);
12555 VBOXSTRICTRC rcStrict;
12556 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12557 switch (uAccessType)
12558 {
12559 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12560 {
12561 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12562 AssertRCReturn(rc, rc);
12563
12564 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12565 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12566 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12567 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12568 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12569 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12570 {
12571 case 0: /* CR0 */
12572 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12573 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12574 break;
12575 case 2: /* CR2 */
12576 /* Nothing to do here, CR2 it's not part of the VMCS. */
12577 break;
12578 case 3: /* CR3 */
12579 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12580 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12581 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12582 break;
12583 case 4: /* CR4 */
12584 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12585 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12586 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12587 break;
12588 case 8: /* CR8 */
12589 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12590 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12591 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12592 break;
12593 default:
12594 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12595 break;
12596 }
12597
12598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12599 break;
12600 }
12601
12602 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12603 {
12604 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12605 AssertRCReturn(rc, rc);
12606
12607 Assert( !pVM->hm.s.fNestedPaging
12608 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12609 || pVCpu->hm.s.fUsingDebugLoop
12610 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12611
12612 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12613 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12614 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12615
12616 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12617 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12618 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12619 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12620 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12621 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12622 VBOXSTRICTRC_VAL(rcStrict)));
12623 break;
12624 }
12625
12626 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12627 {
12628 AssertRCReturn(rc, rc);
12629 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12630 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12631 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12632 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12633 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12634 break;
12635 }
12636
12637 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12638 {
12639 AssertRCReturn(rc, rc);
12640 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12641 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12642 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12643 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12644 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12646 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12647 break;
12648 }
12649
12650 default:
12651 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12652 VERR_VMX_UNEXPECTED_EXCEPTION);
12653 }
12654
12655 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12656 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12657 NOREF(pVM);
12658 return rcStrict;
12659}
12660
12661
12662/**
12663 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12664 * VM-exit.
12665 */
12666HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12667{
12668 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12669 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12670
12671 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12672 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12673 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12674 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12675 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12676 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12677 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12678 AssertRCReturn(rc, rc);
12679
12680 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12681 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12682 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12683 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12684 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12685 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12686 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12687 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12688 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12689
12690 /* I/O operation lookup arrays. */
12691 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12692 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12693
12694 VBOXSTRICTRC rcStrict;
12695 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12696 uint32_t const cbInstr = pVmxTransient->cbInstr;
12697 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12698 PVM pVM = pVCpu->CTX_SUFF(pVM);
12699 if (fIOString)
12700 {
12701#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12702 See @bugref{5752#c158}. Should work now. */
12703 /*
12704 * INS/OUTS - I/O String instruction.
12705 *
12706 * Use instruction-information if available, otherwise fall back on
12707 * interpreting the instruction.
12708 */
12709 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12710 fIOWrite ? 'w' : 'r'));
12711 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12712 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12713 {
12714 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12715 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12716 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12717 AssertRCReturn(rc2, rc2);
12718 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12719 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12720 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12721 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12722 if (fIOWrite)
12723 {
12724 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12725 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12726 }
12727 else
12728 {
12729 /*
12730 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12731 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12732 * See Intel Instruction spec. for "INS".
12733 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12734 */
12735 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12736 }
12737 }
12738 else
12739 {
12740 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12741 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12742 AssertRCReturn(rc2, rc2);
12743 rcStrict = IEMExecOne(pVCpu);
12744 }
12745 /** @todo IEM needs to be setting these flags somehow. */
12746 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12747 fUpdateRipAlready = true;
12748#else
12749 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12750 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12751 if (RT_SUCCESS(rcStrict))
12752 {
12753 if (fIOWrite)
12754 {
12755 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12756 (DISCPUMODE)pDis->uAddrMode, cbValue);
12757 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12758 }
12759 else
12760 {
12761 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12762 (DISCPUMODE)pDis->uAddrMode, cbValue);
12763 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12764 }
12765 }
12766 else
12767 {
12768 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12769 pMixedCtx->rip));
12770 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12771 }
12772#endif
12773 }
12774 else
12775 {
12776 /*
12777 * IN/OUT - I/O instruction.
12778 */
12779 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12780 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12781 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12782 if (fIOWrite)
12783 {
12784 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12786 }
12787 else
12788 {
12789 uint32_t u32Result = 0;
12790 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12791 if (IOM_SUCCESS(rcStrict))
12792 {
12793 /* Save result of I/O IN instr. in AL/AX/EAX. */
12794 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12795 }
12796 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12797 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12799 }
12800 }
12801
12802 if (IOM_SUCCESS(rcStrict))
12803 {
12804 if (!fUpdateRipAlready)
12805 {
12806 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12807 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12808 }
12809
12810 /*
12811 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12812 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12813 */
12814 if (fIOString)
12815 {
12816 /** @todo Single-step for INS/OUTS with REP prefix? */
12817 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12818 }
12819 else if ( !fDbgStepping
12820 && fGstStepping)
12821 {
12822 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12823 }
12824
12825 /*
12826 * If any I/O breakpoints are armed, we need to check if one triggered
12827 * and take appropriate action.
12828 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12829 */
12830 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12831 AssertRCReturn(rc2, rc2);
12832
12833 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12834 * execution engines about whether hyper BPs and such are pending. */
12835 uint32_t const uDr7 = pMixedCtx->dr[7];
12836 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12837 && X86_DR7_ANY_RW_IO(uDr7)
12838 && (pMixedCtx->cr4 & X86_CR4_DE))
12839 || DBGFBpIsHwIoArmed(pVM)))
12840 {
12841 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12842
12843 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12844 VMMRZCallRing3Disable(pVCpu);
12845 HM_DISABLE_PREEMPT();
12846
12847 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12848
12849 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12850 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12851 {
12852 /* Raise #DB. */
12853 if (fIsGuestDbgActive)
12854 ASMSetDR6(pMixedCtx->dr[6]);
12855 if (pMixedCtx->dr[7] != uDr7)
12856 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12857
12858 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12859 }
12860 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12861 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12862 else if ( rcStrict2 != VINF_SUCCESS
12863 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12864 rcStrict = rcStrict2;
12865 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12866
12867 HM_RESTORE_PREEMPT();
12868 VMMRZCallRing3Enable(pVCpu);
12869 }
12870 }
12871
12872#ifdef VBOX_STRICT
12873 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12874 Assert(!fIOWrite);
12875 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12876 Assert(fIOWrite);
12877 else
12878 {
12879#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12880 * statuses, that the VMM device and some others may return. See
12881 * IOM_SUCCESS() for guidance. */
12882 AssertMsg( RT_FAILURE(rcStrict)
12883 || rcStrict == VINF_SUCCESS
12884 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12885 || rcStrict == VINF_EM_DBG_BREAKPOINT
12886 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12887 || rcStrict == VINF_EM_RAW_TO_R3
12888 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12889#endif
12890 }
12891#endif
12892
12893 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12894 return rcStrict;
12895}
12896
12897
12898/**
12899 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12900 * VM-exit.
12901 */
12902HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12903{
12904 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12905
12906 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12907 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12908 AssertRCReturn(rc, rc);
12909 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12910 {
12911 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12912 AssertRCReturn(rc, rc);
12913 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12914 {
12915 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12916
12917 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12918 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12919
12920 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12921 Assert(!pVCpu->hm.s.Event.fPending);
12922 pVCpu->hm.s.Event.fPending = true;
12923 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12924 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12925 AssertRCReturn(rc, rc);
12926 if (fErrorCodeValid)
12927 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12928 else
12929 pVCpu->hm.s.Event.u32ErrCode = 0;
12930 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12931 && uVector == X86_XCPT_PF)
12932 {
12933 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12934 }
12935
12936 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12938 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12939 }
12940 }
12941
12942 /* Fall back to the interpreter to emulate the task-switch. */
12943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12944 return VERR_EM_INTERPRETER;
12945}
12946
12947
12948/**
12949 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12950 */
12951HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12952{
12953 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12954 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12955 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12956 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12957 AssertRCReturn(rc, rc);
12958 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12959 return VINF_EM_DBG_STEPPED;
12960}
12961
12962
12963/**
12964 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12965 */
12966HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12967{
12968 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12969
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12971
12972 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12973 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12974 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12975 {
12976 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12977 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12978 {
12979 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12980 return VERR_EM_INTERPRETER;
12981 }
12982 }
12983 else
12984 {
12985 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12986 rcStrict1 = VINF_SUCCESS;
12987 return rcStrict1;
12988 }
12989
12990#if 0
12991 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12992 * just sync the whole thing. */
12993 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12994#else
12995 /* Aggressive state sync. for now. */
12996 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12997 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12998 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12999#endif
13000 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13001 AssertRCReturn(rc, rc);
13002
13003 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13004 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13005 VBOXSTRICTRC rcStrict2;
13006 switch (uAccessType)
13007 {
13008 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13009 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13010 {
13011 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13012 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13013 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13014
13015 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13016 GCPhys &= PAGE_BASE_GC_MASK;
13017 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13018 PVM pVM = pVCpu->CTX_SUFF(pVM);
13019 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13020 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13021
13022 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13023 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13024 CPUMCTX2CORE(pMixedCtx), GCPhys);
13025 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13026 if ( rcStrict2 == VINF_SUCCESS
13027 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13028 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13029 {
13030 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13031 | HM_CHANGED_GUEST_RSP
13032 | HM_CHANGED_GUEST_RFLAGS
13033 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13034 rcStrict2 = VINF_SUCCESS;
13035 }
13036 break;
13037 }
13038
13039 default:
13040 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13041 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13042 break;
13043 }
13044
13045 if (rcStrict2 != VINF_SUCCESS)
13046 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13047 return rcStrict2;
13048}
13049
13050
13051/**
13052 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13053 * VM-exit.
13054 */
13055HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13056{
13057 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13058
13059 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13060 if (pVmxTransient->fWasGuestDebugStateActive)
13061 {
13062 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13063 HMVMX_RETURN_UNEXPECTED_EXIT();
13064 }
13065
13066 if ( !pVCpu->hm.s.fSingleInstruction
13067 && !pVmxTransient->fWasHyperDebugStateActive)
13068 {
13069 Assert(!DBGFIsStepping(pVCpu));
13070 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13071
13072 /* Don't intercept MOV DRx any more. */
13073 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13074 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13075 AssertRCReturn(rc, rc);
13076
13077 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13078 VMMRZCallRing3Disable(pVCpu);
13079 HM_DISABLE_PREEMPT();
13080
13081 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13082 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13083 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13084
13085 HM_RESTORE_PREEMPT();
13086 VMMRZCallRing3Enable(pVCpu);
13087
13088#ifdef VBOX_WITH_STATISTICS
13089 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13090 AssertRCReturn(rc, rc);
13091 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13093 else
13094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13095#endif
13096 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13097 return VINF_SUCCESS;
13098 }
13099
13100 /*
13101 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13102 * Update the segment registers and DR7 from the CPU.
13103 */
13104 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13105 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13106 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13107 AssertRCReturn(rc, rc);
13108 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13109
13110 PVM pVM = pVCpu->CTX_SUFF(pVM);
13111 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13112 {
13113 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13114 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13115 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13116 if (RT_SUCCESS(rc))
13117 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13118 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13119 }
13120 else
13121 {
13122 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13123 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13124 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13125 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13126 }
13127
13128 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13129 if (RT_SUCCESS(rc))
13130 {
13131 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13132 AssertRCReturn(rc2, rc2);
13133 return VINF_SUCCESS;
13134 }
13135 return rc;
13136}
13137
13138
13139/**
13140 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13141 * Conditional VM-exit.
13142 */
13143HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13144{
13145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13146 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13147
13148 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13149 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13150 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13151 {
13152 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13153 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13154 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13155 {
13156 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13157 return VERR_EM_INTERPRETER;
13158 }
13159 }
13160 else
13161 {
13162 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13163 rcStrict1 = VINF_SUCCESS;
13164 return rcStrict1;
13165 }
13166
13167 RTGCPHYS GCPhys = 0;
13168 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13169
13170#if 0
13171 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13172#else
13173 /* Aggressive state sync. for now. */
13174 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13175 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13176 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13177#endif
13178 AssertRCReturn(rc, rc);
13179
13180 /*
13181 * If we succeed, resume guest execution.
13182 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13183 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13184 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13185 * weird case. See @bugref{6043}.
13186 */
13187 PVM pVM = pVCpu->CTX_SUFF(pVM);
13188 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13189 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13190 if ( rcStrict2 == VINF_SUCCESS
13191 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13192 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13193 {
13194 /* Successfully handled MMIO operation. */
13195 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13196 | HM_CHANGED_GUEST_RSP
13197 | HM_CHANGED_GUEST_RFLAGS
13198 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13199 return VINF_SUCCESS;
13200 }
13201 return rcStrict2;
13202}
13203
13204
13205/**
13206 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13207 * VM-exit.
13208 */
13209HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13210{
13211 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13212 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13213
13214 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13215 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13216 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13217 {
13218 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13219 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13220 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13221 }
13222 else
13223 {
13224 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13225 rcStrict1 = VINF_SUCCESS;
13226 return rcStrict1;
13227 }
13228
13229 RTGCPHYS GCPhys = 0;
13230 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13231 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13232#if 0
13233 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13234#else
13235 /* Aggressive state sync. for now. */
13236 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13237 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13238 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13239#endif
13240 AssertRCReturn(rc, rc);
13241
13242 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13243 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13244
13245 RTGCUINT uErrorCode = 0;
13246 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13247 uErrorCode |= X86_TRAP_PF_ID;
13248 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13249 uErrorCode |= X86_TRAP_PF_RW;
13250 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13251 uErrorCode |= X86_TRAP_PF_P;
13252
13253 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13254
13255 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13256 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13257
13258 /* Handle the pagefault trap for the nested shadow table. */
13259 PVM pVM = pVCpu->CTX_SUFF(pVM);
13260 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13261 TRPMResetTrap(pVCpu);
13262
13263 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13264 if ( rcStrict2 == VINF_SUCCESS
13265 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13266 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13267 {
13268 /* Successfully synced our nested page tables. */
13269 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13270 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13271 | HM_CHANGED_GUEST_RSP
13272 | HM_CHANGED_GUEST_RFLAGS);
13273 return VINF_SUCCESS;
13274 }
13275
13276 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13277 return rcStrict2;
13278}
13279
13280/** @} */
13281
13282/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13283/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13284/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13285
13286/** @name VM-exit exception handlers.
13287 * @{
13288 */
13289
13290/**
13291 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13292 */
13293static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13294{
13295 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13297
13298 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13299 AssertRCReturn(rc, rc);
13300
13301 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13302 {
13303 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13304 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13305
13306 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13307 * provides VM-exit instruction length. If this causes problem later,
13308 * disassemble the instruction like it's done on AMD-V. */
13309 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13310 AssertRCReturn(rc2, rc2);
13311 return rc;
13312 }
13313
13314 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13315 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13316 return rc;
13317}
13318
13319
13320/**
13321 * VM-exit exception handler for \#BP (Breakpoint exception).
13322 */
13323static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13324{
13325 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13326 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13327
13328 /** @todo Try optimize this by not saving the entire guest state unless
13329 * really needed. */
13330 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13331 AssertRCReturn(rc, rc);
13332
13333 PVM pVM = pVCpu->CTX_SUFF(pVM);
13334 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13335 if (rc == VINF_EM_RAW_GUEST_TRAP)
13336 {
13337 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13338 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13339 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13340 AssertRCReturn(rc, rc);
13341
13342 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13343 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13344 }
13345
13346 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13347 return rc;
13348}
13349
13350
13351/**
13352 * VM-exit exception handler for \#AC (alignment check exception).
13353 */
13354static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13355{
13356 RT_NOREF_PV(pMixedCtx);
13357 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13358
13359 /*
13360 * Re-inject it. We'll detect any nesting before getting here.
13361 */
13362 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13363 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13364 AssertRCReturn(rc, rc);
13365 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13366
13367 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13368 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13369 return VINF_SUCCESS;
13370}
13371
13372
13373/**
13374 * VM-exit exception handler for \#DB (Debug exception).
13375 */
13376static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13377{
13378 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13380 Log6(("XcptDB\n"));
13381
13382 /*
13383 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13384 * for processing.
13385 */
13386 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13387 AssertRCReturn(rc, rc);
13388
13389 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13390 uint64_t uDR6 = X86_DR6_INIT_VAL;
13391 uDR6 |= ( pVmxTransient->uExitQualification
13392 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13393
13394 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13395 if (rc == VINF_EM_RAW_GUEST_TRAP)
13396 {
13397 /*
13398 * The exception was for the guest. Update DR6, DR7.GD and
13399 * IA32_DEBUGCTL.LBR before forwarding it.
13400 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13401 */
13402 VMMRZCallRing3Disable(pVCpu);
13403 HM_DISABLE_PREEMPT();
13404
13405 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13406 pMixedCtx->dr[6] |= uDR6;
13407 if (CPUMIsGuestDebugStateActive(pVCpu))
13408 ASMSetDR6(pMixedCtx->dr[6]);
13409
13410 HM_RESTORE_PREEMPT();
13411 VMMRZCallRing3Enable(pVCpu);
13412
13413 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13414 AssertRCReturn(rc, rc);
13415
13416 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13417 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13418
13419 /* Paranoia. */
13420 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13421 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13422
13423 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13424 AssertRCReturn(rc, rc);
13425
13426 /*
13427 * Raise #DB in the guest.
13428 *
13429 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13430 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13431 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13432 *
13433 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13434 */
13435 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13436 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13437 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13438 AssertRCReturn(rc, rc);
13439 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13440 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13441 return VINF_SUCCESS;
13442 }
13443
13444 /*
13445 * Not a guest trap, must be a hypervisor related debug event then.
13446 * Update DR6 in case someone is interested in it.
13447 */
13448 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13449 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13450 CPUMSetHyperDR6(pVCpu, uDR6);
13451
13452 return rc;
13453}
13454
13455
13456/**
13457 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13458 * point exception).
13459 */
13460static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13461{
13462 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13463
13464 /* We require CR0 and EFER. EFER is always up-to-date. */
13465 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13466 AssertRCReturn(rc, rc);
13467
13468 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13469 VMMRZCallRing3Disable(pVCpu);
13470 HM_DISABLE_PREEMPT();
13471
13472 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13473 if (pVmxTransient->fWasGuestFPUStateActive)
13474 {
13475 rc = VINF_EM_RAW_GUEST_TRAP;
13476 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13477 }
13478 else
13479 {
13480#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13481 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13482#endif
13483 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13484 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13485 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13486 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13487 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13488 }
13489
13490 HM_RESTORE_PREEMPT();
13491 VMMRZCallRing3Enable(pVCpu);
13492
13493 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13494 {
13495 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13496 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13498 pVCpu->hm.s.fPreloadGuestFpu = true;
13499 }
13500 else
13501 {
13502 /* Forward #NM to the guest. */
13503 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13504 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13505 AssertRCReturn(rc, rc);
13506 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13507 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13508 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13509 }
13510
13511 return VINF_SUCCESS;
13512}
13513
13514
13515/**
13516 * VM-exit exception handler for \#GP (General-protection exception).
13517 *
13518 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13519 */
13520static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13521{
13522 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13524
13525 int rc;
13526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13527 { /* likely */ }
13528 else
13529 {
13530#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13531 Assert(pVCpu->hm.s.fUsingDebugLoop);
13532#endif
13533 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13534 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13535 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13536 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13537 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13538 AssertRCReturn(rc, rc);
13539 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13540 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13541 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13542 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13543 return rc;
13544 }
13545
13546 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13547 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13548
13549 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13550 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13551 AssertRCReturn(rc, rc);
13552
13553 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13554 uint32_t cbOp = 0;
13555 PVM pVM = pVCpu->CTX_SUFF(pVM);
13556 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13557 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13558 if (RT_SUCCESS(rc))
13559 {
13560 rc = VINF_SUCCESS;
13561 Assert(cbOp == pDis->cbInstr);
13562 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13563 switch (pDis->pCurInstr->uOpcode)
13564 {
13565 case OP_CLI:
13566 {
13567 pMixedCtx->eflags.Bits.u1IF = 0;
13568 pMixedCtx->eflags.Bits.u1RF = 0;
13569 pMixedCtx->rip += pDis->cbInstr;
13570 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13571 if ( !fDbgStepping
13572 && pMixedCtx->eflags.Bits.u1TF)
13573 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13574 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13575 break;
13576 }
13577
13578 case OP_STI:
13579 {
13580 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13581 pMixedCtx->eflags.Bits.u1IF = 1;
13582 pMixedCtx->eflags.Bits.u1RF = 0;
13583 pMixedCtx->rip += pDis->cbInstr;
13584 if (!fOldIF)
13585 {
13586 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13587 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13588 }
13589 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13590 if ( !fDbgStepping
13591 && pMixedCtx->eflags.Bits.u1TF)
13592 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13593 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13594 break;
13595 }
13596
13597 case OP_HLT:
13598 {
13599 rc = VINF_EM_HALT;
13600 pMixedCtx->rip += pDis->cbInstr;
13601 pMixedCtx->eflags.Bits.u1RF = 0;
13602 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13603 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13604 break;
13605 }
13606
13607 case OP_POPF:
13608 {
13609 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13610 uint32_t cbParm;
13611 uint32_t uMask;
13612 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13613 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13614 {
13615 cbParm = 4;
13616 uMask = 0xffffffff;
13617 }
13618 else
13619 {
13620 cbParm = 2;
13621 uMask = 0xffff;
13622 }
13623
13624 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13625 RTGCPTR GCPtrStack = 0;
13626 X86EFLAGS Eflags;
13627 Eflags.u32 = 0;
13628 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13629 &GCPtrStack);
13630 if (RT_SUCCESS(rc))
13631 {
13632 Assert(sizeof(Eflags.u32) >= cbParm);
13633 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13634 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13635 }
13636 if (RT_FAILURE(rc))
13637 {
13638 rc = VERR_EM_INTERPRETER;
13639 break;
13640 }
13641 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13642 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13643 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13644 pMixedCtx->esp += cbParm;
13645 pMixedCtx->esp &= uMask;
13646 pMixedCtx->rip += pDis->cbInstr;
13647 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13648 | HM_CHANGED_GUEST_RSP
13649 | HM_CHANGED_GUEST_RFLAGS);
13650 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13651 POPF restores EFLAGS.TF. */
13652 if ( !fDbgStepping
13653 && fGstStepping)
13654 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13656 break;
13657 }
13658
13659 case OP_PUSHF:
13660 {
13661 uint32_t cbParm;
13662 uint32_t uMask;
13663 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13664 {
13665 cbParm = 4;
13666 uMask = 0xffffffff;
13667 }
13668 else
13669 {
13670 cbParm = 2;
13671 uMask = 0xffff;
13672 }
13673
13674 /* Get the stack pointer & push the contents of eflags onto the stack. */
13675 RTGCPTR GCPtrStack = 0;
13676 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13677 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13678 if (RT_FAILURE(rc))
13679 {
13680 rc = VERR_EM_INTERPRETER;
13681 break;
13682 }
13683 X86EFLAGS Eflags = pMixedCtx->eflags;
13684 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13685 Eflags.Bits.u1RF = 0;
13686 Eflags.Bits.u1VM = 0;
13687
13688 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13689 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13690 {
13691 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13692 rc = VERR_EM_INTERPRETER;
13693 break;
13694 }
13695 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13696 pMixedCtx->esp -= cbParm;
13697 pMixedCtx->esp &= uMask;
13698 pMixedCtx->rip += pDis->cbInstr;
13699 pMixedCtx->eflags.Bits.u1RF = 0;
13700 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13701 | HM_CHANGED_GUEST_RSP
13702 | HM_CHANGED_GUEST_RFLAGS);
13703 if ( !fDbgStepping
13704 && pMixedCtx->eflags.Bits.u1TF)
13705 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13706 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13707 break;
13708 }
13709
13710 case OP_IRET:
13711 {
13712 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13713 * instruction reference. */
13714 RTGCPTR GCPtrStack = 0;
13715 uint32_t uMask = 0xffff;
13716 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13717 uint16_t aIretFrame[3];
13718 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13719 {
13720 rc = VERR_EM_INTERPRETER;
13721 break;
13722 }
13723 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13724 &GCPtrStack);
13725 if (RT_SUCCESS(rc))
13726 {
13727 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13728 PGMACCESSORIGIN_HM));
13729 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13730 }
13731 if (RT_FAILURE(rc))
13732 {
13733 rc = VERR_EM_INTERPRETER;
13734 break;
13735 }
13736 pMixedCtx->eip = 0;
13737 pMixedCtx->ip = aIretFrame[0];
13738 pMixedCtx->cs.Sel = aIretFrame[1];
13739 pMixedCtx->cs.ValidSel = aIretFrame[1];
13740 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13741 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13742 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13743 pMixedCtx->sp += sizeof(aIretFrame);
13744 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13745 | HM_CHANGED_GUEST_SEGMENT_REGS
13746 | HM_CHANGED_GUEST_RSP
13747 | HM_CHANGED_GUEST_RFLAGS);
13748 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13749 if ( !fDbgStepping
13750 && fGstStepping)
13751 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13752 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13753 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13754 break;
13755 }
13756
13757 case OP_INT:
13758 {
13759 uint16_t uVector = pDis->Param1.uValue & 0xff;
13760 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13761 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13762 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13763 break;
13764 }
13765
13766 case OP_INTO:
13767 {
13768 if (pMixedCtx->eflags.Bits.u1OF)
13769 {
13770 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13771 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13772 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13773 }
13774 else
13775 {
13776 pMixedCtx->eflags.Bits.u1RF = 0;
13777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13778 }
13779 break;
13780 }
13781
13782 default:
13783 {
13784 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13785 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13786 EMCODETYPE_SUPERVISOR);
13787 rc = VBOXSTRICTRC_VAL(rc2);
13788 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13789 /** @todo We have to set pending-debug exceptions here when the guest is
13790 * single-stepping depending on the instruction that was interpreted. */
13791 Log4(("#GP rc=%Rrc\n", rc));
13792 break;
13793 }
13794 }
13795 }
13796 else
13797 rc = VERR_EM_INTERPRETER;
13798
13799 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13800 ("#GP Unexpected rc=%Rrc\n", rc));
13801 return rc;
13802}
13803
13804
13805/**
13806 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13807 * the exception reported in the VMX transient structure back into the VM.
13808 *
13809 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13810 * up-to-date.
13811 */
13812static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13813{
13814 RT_NOREF_PV(pMixedCtx);
13815 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13816#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13817 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13818 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13819 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13820#endif
13821
13822 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13823 hmR0VmxCheckExitDueToEventDelivery(). */
13824 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13825 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13826 AssertRCReturn(rc, rc);
13827 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13828
13829#ifdef DEBUG_ramshankar
13830 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13831 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13832 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13833#endif
13834
13835 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13836 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13837 return VINF_SUCCESS;
13838}
13839
13840
13841/**
13842 * VM-exit exception handler for \#PF (Page-fault exception).
13843 */
13844static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13845{
13846 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13847 PVM pVM = pVCpu->CTX_SUFF(pVM);
13848 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13849 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13850 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13851 AssertRCReturn(rc, rc);
13852
13853 if (!pVM->hm.s.fNestedPaging)
13854 { /* likely */ }
13855 else
13856 {
13857#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13858 Assert(pVCpu->hm.s.fUsingDebugLoop);
13859#endif
13860 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13861 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13862 {
13863 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13864 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13865 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13866 }
13867 else
13868 {
13869 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13870 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13871 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13872 }
13873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13874 return rc;
13875 }
13876
13877 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13878 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13879 if (pVmxTransient->fVectoringPF)
13880 {
13881 Assert(pVCpu->hm.s.Event.fPending);
13882 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13883 }
13884
13885 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13886 AssertRCReturn(rc, rc);
13887
13888 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13889 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13890
13891 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13892 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13893 (RTGCPTR)pVmxTransient->uExitQualification);
13894
13895 Log4(("#PF: rc=%Rrc\n", rc));
13896 if (rc == VINF_SUCCESS)
13897 {
13898#if 0
13899 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13900 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13901 * memory? We don't update the whole state here... */
13902 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13903 | HM_CHANGED_GUEST_RSP
13904 | HM_CHANGED_GUEST_RFLAGS
13905 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13906#else
13907 /*
13908 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13909 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13910 */
13911 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13912 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13913#endif
13914 TRPMResetTrap(pVCpu);
13915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13916 return rc;
13917 }
13918
13919 if (rc == VINF_EM_RAW_GUEST_TRAP)
13920 {
13921 if (!pVmxTransient->fVectoringDoublePF)
13922 {
13923 /* It's a guest page fault and needs to be reflected to the guest. */
13924 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13925 TRPMResetTrap(pVCpu);
13926 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13927 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13928 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13929 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13930 }
13931 else
13932 {
13933 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13934 TRPMResetTrap(pVCpu);
13935 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13936 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13937 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13938 }
13939
13940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13941 return VINF_SUCCESS;
13942 }
13943
13944 TRPMResetTrap(pVCpu);
13945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13946 return rc;
13947}
13948
13949/** @} */
13950
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette