VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65233

Last change on this file since 65233 was 65233, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: column limit.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 585.0 KB
Line 
1/* $Id: HMVMXR0.cpp 65233 2017-01-11 07:29:44Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Saves a set of guest MSRs back into the guest-CPU context.
1524 *
1525 * @param pVCpu The cross context virtual CPU structure.
1526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527 * out-of-sync. Make sure to update the required fields
1528 * before using them.
1529 *
1530 * @remarks No-long-jump zone!!!
1531 */
1532static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533{
1534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538 {
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547 }
1548#else
1549 NOREF(pMixedCtx);
1550#endif
1551 }
1552}
1553
1554
1555/**
1556 * Loads a set of guests MSRs to allow read/passthru to the guest.
1557 *
1558 * The name of this function is slightly confusing. This function does NOT
1559 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560 * common prefix for functions dealing with "lazy restoration" of the shared
1561 * MSRs.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565 * out-of-sync. Make sure to update the required fields
1566 * before using them.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1576 do { \
1577 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1578 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1579 else \
1580 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1581 } while (0)
1582
1583 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1584 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1585 {
1586#if HC_ARCH_BITS == 64
1587 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1588 {
1589 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1590 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1593 }
1594#else
1595 RT_NOREF(pMixedCtx);
1596#endif
1597 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1598 }
1599
1600#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1601}
1602
1603
1604/**
1605 * Performs lazy restoration of the set of host MSRs if they were previously
1606 * loaded with guest MSR values.
1607 *
1608 * @param pVCpu The cross context virtual CPU structure.
1609 *
1610 * @remarks No-long-jump zone!!!
1611 * @remarks The guest MSRs should have been saved back into the guest-CPU
1612 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1613 */
1614static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1615{
1616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1618
1619 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1620 {
1621 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1622#if HC_ARCH_BITS == 64
1623 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1624 {
1625 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1626 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1627 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1628 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1629 }
1630#endif
1631 }
1632 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1633}
1634
1635
1636/**
1637 * Verifies that our cached values of the VMCS controls are all
1638 * consistent with what's actually present in the VMCS.
1639 *
1640 * @returns VBox status code.
1641 * @param pVCpu The cross context virtual CPU structure.
1642 */
1643static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1644{
1645 uint32_t u32Val;
1646 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1649 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1654 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1659 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1662 AssertRCReturn(rc, rc);
1663 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1664 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1665
1666 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1667 {
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1671 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1672 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1673 }
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmFlush.
1766 * @param enmFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1787 rc));
1788 if ( RT_SUCCESS(rc)
1789 && pVCpu)
1790 {
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792 }
1793}
1794
1795
1796/**
1797 * Flushes the TLB using VPID.
1798 *
1799 * @returns VBox status code.
1800 * @param pVM The cross context VM structure.
1801 * @param pVCpu The cross context virtual CPU structure of the calling
1802 * EMT. Can be NULL depending on @a enmFlush.
1803 * @param enmFlush Type of flush.
1804 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1805 * on @a enmFlush).
1806 *
1807 * @remarks Can be called with interrupts disabled.
1808 */
1809static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1810{
1811 NOREF(pVM);
1812 AssertPtr(pVM);
1813 Assert(pVM->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833 if ( RT_SUCCESS(rc)
1834 && pVCpu)
1835 {
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 }
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVM The cross context VM structure.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVM);
1853 AssertPtr(pVCpu);
1854 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1855
1856 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1857 if (!fFlushPending)
1858 {
1859 /*
1860 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1861 * See @bugref{6043} and @bugref{6177}.
1862 *
1863 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1864 * function maybe called in a loop with individual addresses.
1865 */
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1869 {
1870 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1871 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1872 }
1873 else
1874 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1875 }
1876 else if (pVM->hm.s.fNestedPaging)
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 }
1879
1880 return VINF_SUCCESS;
1881}
1882
1883
1884/**
1885 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1886 * otherwise there is nothing really to invalidate.
1887 *
1888 * @returns VBox status code.
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param GCPhys Guest physical address of the page to invalidate.
1892 */
1893VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1894{
1895 NOREF(pVM); NOREF(GCPhys);
1896 LogFlowFunc(("%RGp\n", GCPhys));
1897
1898 /*
1899 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1900 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1901 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1902 */
1903 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1911 * case where neither EPT nor VPID is supported by the CPU.
1912 *
1913 * @param pVM The cross context VM structure.
1914 * @param pVCpu The cross context virtual CPU structure.
1915 * @param pCpu Pointer to the global HM struct.
1916 *
1917 * @remarks Called with interrupts disabled.
1918 */
1919static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1920{
1921 AssertPtr(pVCpu);
1922 AssertPtr(pCpu);
1923 NOREF(pVM);
1924
1925 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1926
1927 Assert(pCpu->idCpu != NIL_RTCPUID);
1928 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1929 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1930 pVCpu->hm.s.fForceTLBFlush = false;
1931 return;
1932}
1933
1934
1935/**
1936 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1937 *
1938 * @param pVM The cross context VM structure.
1939 * @param pVCpu The cross context virtual CPU structure.
1940 * @param pCpu Pointer to the global HM CPU struct.
1941 * @remarks All references to "ASID" in this function pertains to "VPID" in
1942 * Intel's nomenclature. The reason is, to avoid confusion in compare
1943 * statements since the host-CPU copies are named "ASID".
1944 *
1945 * @remarks Called with interrupts disabled.
1946 */
1947static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1948{
1949#ifdef VBOX_WITH_STATISTICS
1950 bool fTlbFlushed = false;
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1953 if (!fTlbFlushed) \
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1955 } while (0)
1956#else
1957# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1958# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1959#endif
1960
1961 AssertPtr(pVM);
1962 AssertPtr(pCpu);
1963 AssertPtr(pVCpu);
1964 Assert(pCpu->idCpu != NIL_RTCPUID);
1965
1966 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1967 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1968 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1969
1970 /*
1971 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1972 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1973 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1974 */
1975 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1976 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1977 {
1978 ++pCpu->uCurrentAsid;
1979 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1980 {
1981 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1982 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1983 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1984 }
1985
1986 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1987 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1988 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1989
1990 /*
1991 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1992 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1993 */
1994 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1995 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1996 HMVMX_SET_TAGGED_TLB_FLUSHED();
1997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1998 }
1999
2000 /* Check for explicit TLB flushes. */
2001 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2002 {
2003 /*
2004 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2005 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2006 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2007 * but not guest-physical mappings.
2008 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2009 */
2010 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2011 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2012 HMVMX_SET_TAGGED_TLB_FLUSHED();
2013 }
2014
2015 pVCpu->hm.s.fForceTLBFlush = false;
2016 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2017
2018 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2019 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2020 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2021 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2022 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2023 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2024 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2025 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2026 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2027
2028 /* Update VMCS with the VPID. */
2029 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2030 AssertRC(rc);
2031
2032#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2033}
2034
2035
2036/**
2037 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2038 *
2039 * @returns VBox status code.
2040 * @param pVM The cross context VM structure.
2041 * @param pVCpu The cross context virtual CPU structure.
2042 * @param pCpu Pointer to the global HM CPU struct.
2043 *
2044 * @remarks Called with interrupts disabled.
2045 */
2046static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2047{
2048 AssertPtr(pVM);
2049 AssertPtr(pVCpu);
2050 AssertPtr(pCpu);
2051 Assert(pCpu->idCpu != NIL_RTCPUID);
2052 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2053 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2054
2055 /*
2056 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2057 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 pVCpu->hm.s.fForceTLBFlush = true;
2070 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2071 }
2072
2073 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2074 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2075
2076 if (pVCpu->hm.s.fForceTLBFlush)
2077 {
2078 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2079 pVCpu->hm.s.fForceTLBFlush = false;
2080 }
2081}
2082
2083
2084/**
2085 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2086 *
2087 * @returns VBox status code.
2088 * @param pVM The cross context VM structure.
2089 * @param pVCpu The cross context virtual CPU structure.
2090 * @param pCpu Pointer to the global HM CPU struct.
2091 *
2092 * @remarks Called with interrupts disabled.
2093 */
2094static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2095{
2096 AssertPtr(pVM);
2097 AssertPtr(pVCpu);
2098 AssertPtr(pCpu);
2099 Assert(pCpu->idCpu != NIL_RTCPUID);
2100 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2101 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2102
2103 /*
2104 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2105 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2106 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2107 */
2108 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2109 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2110 {
2111 pVCpu->hm.s.fForceTLBFlush = true;
2112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2113 }
2114
2115 /* Check for explicit TLB flushes. */
2116 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2117 {
2118 /*
2119 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2120 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2121 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2122 */
2123 pVCpu->hm.s.fForceTLBFlush = true;
2124 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2125 }
2126
2127 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2128 if (pVCpu->hm.s.fForceTLBFlush)
2129 {
2130 ++pCpu->uCurrentAsid;
2131 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2132 {
2133 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2134 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2135 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2136 }
2137
2138 pVCpu->hm.s.fForceTLBFlush = false;
2139 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2140 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2141 if (pCpu->fFlushAsidBeforeUse)
2142 {
2143 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2145 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2146 {
2147 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2148 pCpu->fFlushAsidBeforeUse = false;
2149 }
2150 else
2151 {
2152 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2153 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2154 }
2155 }
2156 }
2157
2158 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2159 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2160 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2161 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2163 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2164 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2165
2166 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2167 AssertRC(rc);
2168}
2169
2170
2171/**
2172 * Flushes the guest TLB entry based on CPU capabilities.
2173 *
2174 * @param pVCpu The cross context virtual CPU structure.
2175 * @param pCpu Pointer to the global HM CPU struct.
2176 */
2177DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2178{
2179#ifdef HMVMX_ALWAYS_FLUSH_TLB
2180 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2181#endif
2182 PVM pVM = pVCpu->CTX_SUFF(pVM);
2183 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2184 {
2185 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2186 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2187 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2189 default:
2190 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2191 break;
2192 }
2193
2194 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2195}
2196
2197
2198/**
2199 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2200 * TLB entries from the host TLB before VM-entry.
2201 *
2202 * @returns VBox status code.
2203 * @param pVM The cross context VM structure.
2204 */
2205static int hmR0VmxSetupTaggedTlb(PVM pVM)
2206{
2207 /*
2208 * Determine optimal flush type for Nested Paging.
2209 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2210 * guest execution (see hmR3InitFinalizeR0()).
2211 */
2212 if (pVM->hm.s.fNestedPaging)
2213 {
2214 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2218 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2220 else
2221 {
2222 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2223 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227
2228 /* Make sure the write-back cacheable memory type for EPT is supported. */
2229 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2230 {
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235
2236 /* EPT requires a page-walk length of 4. */
2237 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2238 {
2239 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2240 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2241 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2242 }
2243 }
2244 else
2245 {
2246 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2248 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2249 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2250 }
2251 }
2252
2253 /*
2254 * Determine optimal flush type for VPID.
2255 */
2256 if (pVM->hm.s.vmx.fVpid)
2257 {
2258 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2261 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2262 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2264 else
2265 {
2266 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2267 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2268 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2271 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2272 pVM->hm.s.vmx.fVpid = false;
2273 }
2274 }
2275 else
2276 {
2277 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2278 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2280 pVM->hm.s.vmx.fVpid = false;
2281 }
2282 }
2283
2284 /*
2285 * Setup the handler for flushing tagged-TLBs.
2286 */
2287 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2289 else if (pVM->hm.s.fNestedPaging)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2291 else if (pVM->hm.s.vmx.fVpid)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2293 else
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2295 return VINF_SUCCESS;
2296}
2297
2298
2299/**
2300 * Sets up pin-based VM-execution controls in the VMCS.
2301 *
2302 * @returns VBox status code.
2303 * @param pVM The cross context VM structure.
2304 * @param pVCpu The cross context virtual CPU structure.
2305 */
2306static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2307{
2308 AssertPtr(pVM);
2309 AssertPtr(pVCpu);
2310
2311 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2312 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2313
2314 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2315 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2316
2317 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2319
2320 /* Enable the VMX preemption timer. */
2321 if (pVM->hm.s.vmx.fUsePreemptTimer)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2325 }
2326
2327#if 0
2328 /* Enable posted-interrupt processing. */
2329 if (pVM->hm.s.fPostedIntrs)
2330 {
2331 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2332 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2334 }
2335#endif
2336
2337 if ((val & zap) != val)
2338 {
2339 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2340 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2341 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2342 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2343 }
2344
2345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2346 AssertRCReturn(rc, rc);
2347
2348 pVCpu->hm.s.vmx.u32PinCtls = val;
2349 return rc;
2350}
2351
2352
2353/**
2354 * Sets up processor-based VM-execution controls in the VMCS.
2355 *
2356 * @returns VBox status code.
2357 * @param pVM The cross context VM structure.
2358 * @param pVCpu The cross context virtual CPU structure.
2359 */
2360static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2361{
2362 AssertPtr(pVM);
2363 AssertPtr(pVCpu);
2364
2365 int rc = VERR_INTERNAL_ERROR_5;
2366 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2367 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2368
2369 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2376
2377 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2378 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2379 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2380 {
2381 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2382 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2383 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2384 }
2385
2386 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2387 if (!pVM->hm.s.fNestedPaging)
2388 {
2389 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2390 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2391 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2392 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2393 }
2394
2395 /* Use TPR shadowing if supported by the CPU. */
2396 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2397 {
2398 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2399 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2400 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2401 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2402 AssertRCReturn(rc, rc);
2403
2404 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2405 /* CR8 writes cause a VM-exit based on TPR threshold. */
2406 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2407 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2408 }
2409 else
2410 {
2411 /*
2412 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2413 * Set this control only for 64-bit guests.
2414 */
2415 if (pVM->hm.s.fAllow64BitGuests)
2416 {
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2419 }
2420 }
2421
2422 /* Use MSR-bitmaps if supported by the CPU. */
2423 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2426
2427 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2428 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2429 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 AssertRCReturn(rc, rc);
2431
2432 /*
2433 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2434 * automatically using dedicated fields in the VMCS.
2435 */
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441
2442#if HC_ARCH_BITS == 64
2443 /*
2444 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2445 */
2446 if (pVM->hm.s.fAllow64BitGuests)
2447 {
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 }
2453#endif
2454 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2455 }
2456
2457 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2458 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2459 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2460
2461 if ((val & zap) != val)
2462 {
2463 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2464 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2465 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2466 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2467 }
2468
2469 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2470 AssertRCReturn(rc, rc);
2471
2472 pVCpu->hm.s.vmx.u32ProcCtls = val;
2473
2474 /*
2475 * Secondary processor-based VM-execution controls.
2476 */
2477 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2478 {
2479 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2480 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2481
2482 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2483 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2484
2485 if (pVM->hm.s.fNestedPaging)
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2487 else
2488 {
2489 /*
2490 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2491 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2492 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2493 */
2494 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2496 }
2497
2498 if (pVM->hm.s.vmx.fVpid)
2499 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2500
2501 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2503
2504#if 0
2505 if (pVM->hm.s.fVirtApicRegs)
2506 {
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2509
2510 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2512 }
2513#endif
2514
2515 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2516 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2517 * done dynamically. */
2518 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2519 {
2520 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2521 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2523 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2528 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2529
2530 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2531 && pVM->hm.s.vmx.cPleGapTicks
2532 && pVM->hm.s.vmx.cPleWindowTicks)
2533 {
2534 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2537 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2538 AssertRCReturn(rc, rc);
2539 }
2540
2541 if ((val & zap) != val)
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2544 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2550 AssertRCReturn(rc, rc);
2551
2552 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2553 }
2554 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2555 {
2556 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2557 "available\n"));
2558 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2559 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560 }
2561
2562 return VINF_SUCCESS;
2563}
2564
2565
2566/**
2567 * Sets up miscellaneous (everything other than Pin & Processor-based
2568 * VM-execution) control fields in the VMCS.
2569 *
2570 * @returns VBox status code.
2571 * @param pVM The cross context VM structure.
2572 * @param pVCpu The cross context virtual CPU structure.
2573 */
2574static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2575{
2576 NOREF(pVM);
2577 AssertPtr(pVM);
2578 AssertPtr(pVCpu);
2579
2580 int rc = VERR_GENERAL_FAILURE;
2581
2582 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2583#if 0
2584 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2586 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2587
2588 /*
2589 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2590 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2591 * We thus use the exception bitmap to control it rather than use both.
2592 */
2593 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2594 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2595
2596 /** @todo Explore possibility of using IO-bitmaps. */
2597 /* All IO & IOIO instructions cause VM-exits. */
2598 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2599 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2600
2601 /* Initialize the MSR-bitmap area. */
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2603 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2605 AssertRCReturn(rc, rc);
2606#endif
2607
2608 /* Setup MSR auto-load/store area. */
2609 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2611 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2617 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 AssertRCReturn(rc, rc);
2619
2620 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2622 AssertRCReturn(rc, rc);
2623
2624 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2625#if 0
2626 /* Setup debug controls */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2628 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2629 AssertRCReturn(rc, rc);
2630#endif
2631
2632 return rc;
2633}
2634
2635
2636/**
2637 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2638 *
2639 * We shall setup those exception intercepts that don't change during the
2640 * lifetime of the VM here. The rest are done dynamically while loading the
2641 * guest state.
2642 *
2643 * @returns VBox status code.
2644 * @param pVM The cross context VM structure.
2645 * @param pVCpu The cross context virtual CPU structure.
2646 */
2647static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2648{
2649 AssertPtr(pVM);
2650 AssertPtr(pVCpu);
2651
2652 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2653
2654 uint32_t u32XcptBitmap = 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 *
3406 * @remarks Can cause longjumps!!!
3407 */
3408DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3409{
3410 NOREF(pMixedCtx);
3411
3412 int rc = VINF_SUCCESS;
3413 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3414 {
3415 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3416 && APICIsEnabled(pVCpu))
3417 {
3418 /*
3419 * Setup TPR shadowing.
3420 */
3421 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3422 {
3423 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3424
3425 bool fPendingIntr = false;
3426 uint8_t u8Tpr = 0;
3427 uint8_t u8PendingIntr = 0;
3428 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3429 AssertRCReturn(rc, rc);
3430
3431 /*
3432 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3433 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3434 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3435 */
3436 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3437 uint32_t u32TprThreshold = 0;
3438 if (fPendingIntr)
3439 {
3440 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3441 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3442 const uint8_t u8TprPriority = u8Tpr >> 4;
3443 if (u8PendingPriority <= u8TprPriority)
3444 u32TprThreshold = u8PendingPriority;
3445 }
3446
3447 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3448 AssertRCReturn(rc, rc);
3449 }
3450
3451#ifndef IEM_VERIFICATION_MODE_FULL
3452 /*
3453 * Setup the virtualized-APIC accesses.
3454 *
3455 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
3456 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
3457 */
3458 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3459 {
3460 uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3461 if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3462 {
3463 /* We only care about the APIC base MSR address and not the other bits. */
3464 PVM pVM = pVCpu->CTX_SUFF(pVM);
3465 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3466 RTGCPHYS GCPhysApicBase;
3467 GCPhysApicBase = u64MsrApicBase;
3468 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3469
3470 /*
3471 * We only need a single HC page as the APIC-access page for all VCPUs as it's used
3472 * purely for causing VM-exits and not for data access within the actual page.
3473 *
3474 * The following check ensures we do the mapping on a per-VM basis as our APIC code
3475 * does not allow different APICs to be mapped at different addresses on different VCPUs.
3476 *
3477 * In fact, we do not support remapping of the APIC base at all, see APICSetBaseMsr()
3478 * so we just map this once per-VM.
3479 */
3480 if (ASMAtomicCmpXchgU64(&pVM->hm.s.vmx.GCPhysApicBase, GCPhysApicBase, 0 /* u64Old */))
3481 {
3482 /* Unalias any existing mapping. */
3483 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3484 AssertRCReturn(rc, rc);
3485
3486 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables
3487 if necessary. */
3488 Log4(("HM: VCPU%u: Mapped HC APIC-access page GCPhysApicBase=%#RGp\n", pVCpu->idCpu, GCPhysApicBase));
3489 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
3490 X86_PTE_RW | X86_PTE_P);
3491 AssertRCReturn(rc, rc);
3492 }
3493
3494 /* Update the per-VCPU cache of the APIC base MSR. */
3495 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3496 }
3497 }
3498#endif /* !IEM_VERIFICATION_MODE_FULL */
3499 }
3500 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3501 }
3502
3503 return rc;
3504}
3505
3506
3507/**
3508 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3509 *
3510 * @returns Guest's interruptibility-state.
3511 * @param pVCpu The cross context virtual CPU structure.
3512 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3513 * out-of-sync. Make sure to update the required fields
3514 * before using them.
3515 *
3516 * @remarks No-long-jump zone!!!
3517 */
3518DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3519{
3520 /*
3521 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3522 */
3523 uint32_t uIntrState = 0;
3524 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3525 {
3526 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3527 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3528 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3529 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3530 {
3531 if (pMixedCtx->eflags.Bits.u1IF)
3532 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3533 else
3534 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3535 }
3536 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3537 {
3538 /*
3539 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3540 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3541 */
3542 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3543 }
3544 }
3545
3546 /*
3547 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3548 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3549 * setting this would block host-NMIs and IRET will not clear the blocking.
3550 *
3551 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3552 */
3553 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3554 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3555 {
3556 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3557 }
3558
3559 return uIntrState;
3560}
3561
3562
3563/**
3564 * Loads the guest's interruptibility-state into the guest-state area in the
3565 * VMCS.
3566 *
3567 * @returns VBox status code.
3568 * @param pVCpu The cross context virtual CPU structure.
3569 * @param uIntrState The interruptibility-state to set.
3570 */
3571static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3572{
3573 NOREF(pVCpu);
3574 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3575 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3576 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3577 AssertRC(rc);
3578 return rc;
3579}
3580
3581
3582/**
3583 * Loads the exception intercepts required for guest execution in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu The cross context virtual CPU structure.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 */
3591static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3592{
3593 NOREF(pMixedCtx);
3594 int rc = VINF_SUCCESS;
3595 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3596 {
3597 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3598 if (pVCpu->hm.s.fGIMTrapXcptUD)
3599 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3600#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3601 else
3602 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3603#endif
3604
3605 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3606 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3607
3608 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3609 AssertRCReturn(rc, rc);
3610
3611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3612 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3613 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3614 }
3615 return rc;
3616}
3617
3618
3619/**
3620 * Loads the guest's RIP into the guest-state area in the VMCS.
3621 *
3622 * @returns VBox status code.
3623 * @param pVCpu The cross context virtual CPU structure.
3624 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3625 * out-of-sync. Make sure to update the required fields
3626 * before using them.
3627 *
3628 * @remarks No-long-jump zone!!!
3629 */
3630static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3631{
3632 int rc = VINF_SUCCESS;
3633 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3634 {
3635 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3636 AssertRCReturn(rc, rc);
3637
3638 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3639 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3640 HMCPU_CF_VALUE(pVCpu)));
3641 }
3642 return rc;
3643}
3644
3645
3646/**
3647 * Loads the guest's RSP into the guest-state area in the VMCS.
3648 *
3649 * @returns VBox status code.
3650 * @param pVCpu The cross context virtual CPU structure.
3651 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3652 * out-of-sync. Make sure to update the required fields
3653 * before using them.
3654 *
3655 * @remarks No-long-jump zone!!!
3656 */
3657static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3658{
3659 int rc = VINF_SUCCESS;
3660 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3661 {
3662 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3663 AssertRCReturn(rc, rc);
3664
3665 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3666 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3667 }
3668 return rc;
3669}
3670
3671
3672/**
3673 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3674 *
3675 * @returns VBox status code.
3676 * @param pVCpu The cross context virtual CPU structure.
3677 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3678 * out-of-sync. Make sure to update the required fields
3679 * before using them.
3680 *
3681 * @remarks No-long-jump zone!!!
3682 */
3683static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3684{
3685 int rc = VINF_SUCCESS;
3686 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3687 {
3688 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3689 Let us assert it as such and use 32-bit VMWRITE. */
3690 Assert(!(pMixedCtx->rflags.u64 >> 32));
3691 X86EFLAGS Eflags = pMixedCtx->eflags;
3692 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3693 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3694 * These will never be cleared/set, unless some other part of the VMM
3695 * code is buggy - in which case we're better of finding and fixing
3696 * those bugs than hiding them. */
3697 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3698 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3699 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3700 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3701
3702 /*
3703 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3704 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3705 */
3706 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3707 {
3708 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3709 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3710 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3711 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3712 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3713 }
3714
3715 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3716 AssertRCReturn(rc, rc);
3717
3718 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3719 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3720 }
3721 return rc;
3722}
3723
3724
3725/**
3726 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3727 *
3728 * @returns VBox status code.
3729 * @param pVCpu The cross context virtual CPU structure.
3730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3731 * out-of-sync. Make sure to update the required fields
3732 * before using them.
3733 *
3734 * @remarks No-long-jump zone!!!
3735 */
3736DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3737{
3738 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3739 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3740 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3741 AssertRCReturn(rc, rc);
3742 return rc;
3743}
3744
3745
3746/**
3747 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3748 * CR0 is partially shared with the host and we have to consider the FPU bits.
3749 *
3750 * @returns VBox status code.
3751 * @param pVCpu The cross context virtual CPU structure.
3752 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3753 * out-of-sync. Make sure to update the required fields
3754 * before using them.
3755 *
3756 * @remarks No-long-jump zone!!!
3757 */
3758static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3759{
3760 /*
3761 * Guest CR0.
3762 * Guest FPU.
3763 */
3764 int rc = VINF_SUCCESS;
3765 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3766 {
3767 Assert(!(pMixedCtx->cr0 >> 32));
3768 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3769 PVM pVM = pVCpu->CTX_SUFF(pVM);
3770
3771 /* The guest's view (read access) of its CR0 is unblemished. */
3772 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3773 AssertRCReturn(rc, rc);
3774 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3775
3776 /* Setup VT-x's view of the guest CR0. */
3777 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3778 if (pVM->hm.s.fNestedPaging)
3779 {
3780 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3781 {
3782 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3783 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3784 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3785 }
3786 else
3787 {
3788 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3789 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3790 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3791 }
3792
3793 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3794 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3795 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3796
3797 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3798 AssertRCReturn(rc, rc);
3799 }
3800 else
3801 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3802
3803 /*
3804 * Guest FPU bits.
3805 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3806 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3807 */
3808 u32GuestCR0 |= X86_CR0_NE;
3809 bool fInterceptNM = false;
3810 if (CPUMIsGuestFPUStateActive(pVCpu))
3811 {
3812 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3813 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3814 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3815 }
3816 else
3817 {
3818 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3819 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3820 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3821 }
3822
3823 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3824 bool fInterceptMF = false;
3825 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3826 fInterceptMF = true;
3827
3828 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3829 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3830 {
3831 Assert(PDMVmmDevHeapIsEnabled(pVM));
3832 Assert(pVM->hm.s.vmx.pRealModeTSS);
3833 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3834 fInterceptNM = true;
3835 fInterceptMF = true;
3836 }
3837 else
3838 {
3839 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3840 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3841 }
3842 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3843
3844 if (fInterceptNM)
3845 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3846 else
3847 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3848
3849 if (fInterceptMF)
3850 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3851 else
3852 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3853
3854 /* Additional intercepts for debugging, define these yourself explicitly. */
3855#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3856 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3857 | RT_BIT(X86_XCPT_BP)
3858 | RT_BIT(X86_XCPT_DE)
3859 | RT_BIT(X86_XCPT_NM)
3860 | RT_BIT(X86_XCPT_TS)
3861 | RT_BIT(X86_XCPT_UD)
3862 | RT_BIT(X86_XCPT_NP)
3863 | RT_BIT(X86_XCPT_SS)
3864 | RT_BIT(X86_XCPT_GP)
3865 | RT_BIT(X86_XCPT_PF)
3866 | RT_BIT(X86_XCPT_MF)
3867 ;
3868#elif defined(HMVMX_ALWAYS_TRAP_PF)
3869 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3870#endif
3871
3872 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3873
3874 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3875 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3876 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3877 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3878 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3879 else
3880 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3881
3882 u32GuestCR0 |= uSetCR0;
3883 u32GuestCR0 &= uZapCR0;
3884 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3885
3886 /* Write VT-x's view of the guest CR0 into the VMCS. */
3887 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3888 AssertRCReturn(rc, rc);
3889 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3890 uZapCR0));
3891
3892 /*
3893 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3894 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3895 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3896 */
3897 uint32_t u32CR0Mask = 0;
3898 u32CR0Mask = X86_CR0_PE
3899 | X86_CR0_NE
3900 | X86_CR0_WP
3901 | X86_CR0_PG
3902 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3903 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3904 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3905
3906 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3907 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3908 * and @bugref{6944}. */
3909#if 0
3910 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3911 u32CR0Mask &= ~X86_CR0_PE;
3912#endif
3913 if (pVM->hm.s.fNestedPaging)
3914 u32CR0Mask &= ~X86_CR0_WP;
3915
3916 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3917 if (fInterceptNM)
3918 {
3919 u32CR0Mask |= X86_CR0_TS
3920 | X86_CR0_MP;
3921 }
3922
3923 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3924 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3925 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3926 AssertRCReturn(rc, rc);
3927 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3928
3929 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3930 }
3931 return rc;
3932}
3933
3934
3935/**
3936 * Loads the guest control registers (CR3, CR4) into the guest-state area
3937 * in the VMCS.
3938 *
3939 * @returns VBox strict status code.
3940 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3941 * without unrestricted guest access and the VMMDev is not presently
3942 * mapped (e.g. EFI32).
3943 *
3944 * @param pVCpu The cross context virtual CPU structure.
3945 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3946 * out-of-sync. Make sure to update the required fields
3947 * before using them.
3948 *
3949 * @remarks No-long-jump zone!!!
3950 */
3951static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3952{
3953 int rc = VINF_SUCCESS;
3954 PVM pVM = pVCpu->CTX_SUFF(pVM);
3955
3956 /*
3957 * Guest CR2.
3958 * It's always loaded in the assembler code. Nothing to do here.
3959 */
3960
3961 /*
3962 * Guest CR3.
3963 */
3964 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3965 {
3966 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3967 if (pVM->hm.s.fNestedPaging)
3968 {
3969 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3970
3971 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3972 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3973 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3974 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3975
3976 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3977 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3978 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3979
3980 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3981 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3982 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3983 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3984 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3985 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3986 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3987
3988 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3989 AssertRCReturn(rc, rc);
3990 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3991
3992 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3993 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3994 {
3995 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3996 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3997 {
3998 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3999 AssertRCReturn(rc, rc);
4000 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4001 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4002 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4003 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4004 AssertRCReturn(rc, rc);
4005 }
4006
4007 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4008 have Unrestricted Execution to handle the guest when it's not using paging. */
4009 GCPhysGuestCR3 = pMixedCtx->cr3;
4010 }
4011 else
4012 {
4013 /*
4014 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4015 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4016 * EPT takes care of translating it to host-physical addresses.
4017 */
4018 RTGCPHYS GCPhys;
4019 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4020
4021 /* We obtain it here every time as the guest could have relocated this PCI region. */
4022 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4023 if (RT_SUCCESS(rc))
4024 { /* likely */ }
4025 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4026 {
4027 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4028 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4029 }
4030 else
4031 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4032
4033 GCPhysGuestCR3 = GCPhys;
4034 }
4035
4036 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4037 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4038 }
4039 else
4040 {
4041 /* Non-nested paging case, just use the hypervisor's CR3. */
4042 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4043
4044 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4045 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4046 }
4047 AssertRCReturn(rc, rc);
4048
4049 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4050 }
4051
4052 /*
4053 * Guest CR4.
4054 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4055 */
4056 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4057 {
4058 Assert(!(pMixedCtx->cr4 >> 32));
4059 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4060
4061 /* The guest's view of its CR4 is unblemished. */
4062 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4063 AssertRCReturn(rc, rc);
4064 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4065
4066 /* Setup VT-x's view of the guest CR4. */
4067 /*
4068 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4069 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4070 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4071 */
4072 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4073 {
4074 Assert(pVM->hm.s.vmx.pRealModeTSS);
4075 Assert(PDMVmmDevHeapIsEnabled(pVM));
4076 u32GuestCR4 &= ~X86_CR4_VME;
4077 }
4078
4079 if (pVM->hm.s.fNestedPaging)
4080 {
4081 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4082 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4083 {
4084 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4085 u32GuestCR4 |= X86_CR4_PSE;
4086 /* Our identity mapping is a 32-bit page directory. */
4087 u32GuestCR4 &= ~X86_CR4_PAE;
4088 }
4089 /* else use guest CR4.*/
4090 }
4091 else
4092 {
4093 /*
4094 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4095 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4096 */
4097 switch (pVCpu->hm.s.enmShadowMode)
4098 {
4099 case PGMMODE_REAL: /* Real-mode. */
4100 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4101 case PGMMODE_32_BIT: /* 32-bit paging. */
4102 {
4103 u32GuestCR4 &= ~X86_CR4_PAE;
4104 break;
4105 }
4106
4107 case PGMMODE_PAE: /* PAE paging. */
4108 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4109 {
4110 u32GuestCR4 |= X86_CR4_PAE;
4111 break;
4112 }
4113
4114 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4115 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4116#ifdef VBOX_ENABLE_64_BITS_GUESTS
4117 break;
4118#endif
4119 default:
4120 AssertFailed();
4121 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4122 }
4123 }
4124
4125 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4126 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4127 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4128 u32GuestCR4 |= uSetCR4;
4129 u32GuestCR4 &= uZapCR4;
4130
4131 /* Write VT-x's view of the guest CR4 into the VMCS. */
4132 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4133 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4134 AssertRCReturn(rc, rc);
4135
4136 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4137 uint32_t u32CR4Mask = X86_CR4_VME
4138 | X86_CR4_PAE
4139 | X86_CR4_PGE
4140 | X86_CR4_PSE
4141 | X86_CR4_VMXE;
4142 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4143 u32CR4Mask |= X86_CR4_OSXSAVE;
4144 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4145 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4146 AssertRCReturn(rc, rc);
4147
4148 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4149 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4150
4151 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4152 }
4153 return rc;
4154}
4155
4156
4157/**
4158 * Loads the guest debug registers into the guest-state area in the VMCS.
4159 *
4160 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4161 *
4162 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4163 *
4164 * @returns VBox status code.
4165 * @param pVCpu The cross context virtual CPU structure.
4166 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4167 * out-of-sync. Make sure to update the required fields
4168 * before using them.
4169 *
4170 * @remarks No-long-jump zone!!!
4171 */
4172static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4173{
4174 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4175 return VINF_SUCCESS;
4176
4177#ifdef VBOX_STRICT
4178 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4179 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4180 {
4181 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4182 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4183 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4184 }
4185#endif
4186
4187 int rc;
4188 PVM pVM = pVCpu->CTX_SUFF(pVM);
4189 bool fSteppingDB = false;
4190 bool fInterceptMovDRx = false;
4191 if (pVCpu->hm.s.fSingleInstruction)
4192 {
4193 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4194 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4195 {
4196 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4197 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4198 AssertRCReturn(rc, rc);
4199 Assert(fSteppingDB == false);
4200 }
4201 else
4202 {
4203 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4204 pVCpu->hm.s.fClearTrapFlag = true;
4205 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4206 fSteppingDB = true;
4207 }
4208 }
4209
4210 if ( fSteppingDB
4211 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4212 {
4213 /*
4214 * Use the combined guest and host DRx values found in the hypervisor
4215 * register set because the debugger has breakpoints active or someone
4216 * is single stepping on the host side without a monitor trap flag.
4217 *
4218 * Note! DBGF expects a clean DR6 state before executing guest code.
4219 */
4220#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4221 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4222 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4223 {
4224 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4225 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4226 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4227 }
4228 else
4229#endif
4230 if (!CPUMIsHyperDebugStateActive(pVCpu))
4231 {
4232 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4233 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4234 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4235 }
4236
4237 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4238 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4239 AssertRCReturn(rc, rc);
4240
4241 pVCpu->hm.s.fUsingHyperDR7 = true;
4242 fInterceptMovDRx = true;
4243 }
4244 else
4245 {
4246 /*
4247 * If the guest has enabled debug registers, we need to load them prior to
4248 * executing guest code so they'll trigger at the right time.
4249 */
4250 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4251 {
4252#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4253 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4254 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4255 {
4256 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4257 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4258 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4259 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4260 }
4261 else
4262#endif
4263 if (!CPUMIsGuestDebugStateActive(pVCpu))
4264 {
4265 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4266 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4267 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4268 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4269 }
4270 Assert(!fInterceptMovDRx);
4271 }
4272 /*
4273 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4274 * must intercept #DB in order to maintain a correct DR6 guest value, and
4275 * because we need to intercept it to prevent nested #DBs from hanging the
4276 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4277 */
4278#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4279 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4280 && !CPUMIsGuestDebugStateActive(pVCpu))
4281#else
4282 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4283#endif
4284 {
4285 fInterceptMovDRx = true;
4286 }
4287
4288 /* Update guest DR7. */
4289 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4290 AssertRCReturn(rc, rc);
4291
4292 pVCpu->hm.s.fUsingHyperDR7 = false;
4293 }
4294
4295 /*
4296 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4297 */
4298 if (fInterceptMovDRx)
4299 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4300 else
4301 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4302 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4303 AssertRCReturn(rc, rc);
4304
4305 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4306 return VINF_SUCCESS;
4307}
4308
4309
4310#ifdef VBOX_STRICT
4311/**
4312 * Strict function to validate segment registers.
4313 *
4314 * @remarks ASSUMES CR0 is up to date.
4315 */
4316static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4317{
4318 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4319 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4320 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4321 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4322 && ( !CPUMIsGuestInRealModeEx(pCtx)
4323 && !CPUMIsGuestInV86ModeEx(pCtx)))
4324 {
4325 /* Protected mode checks */
4326 /* CS */
4327 Assert(pCtx->cs.Attr.n.u1Present);
4328 Assert(!(pCtx->cs.Attr.u & 0xf00));
4329 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->cs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4333 || (pCtx->cs.Attr.n.u1Granularity));
4334 /* CS cannot be loaded with NULL in protected mode. */
4335 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4336 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4337 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4338 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4339 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4340 else
4341 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4342 /* SS */
4343 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4344 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4345 if ( !(pCtx->cr0 & X86_CR0_PE)
4346 || pCtx->cs.Attr.n.u4Type == 3)
4347 {
4348 Assert(!pCtx->ss.Attr.n.u2Dpl);
4349 }
4350 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4351 {
4352 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4353 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4354 Assert(pCtx->ss.Attr.n.u1Present);
4355 Assert(!(pCtx->ss.Attr.u & 0xf00));
4356 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4357 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4358 || !(pCtx->ss.Attr.n.u1Granularity));
4359 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4360 || (pCtx->ss.Attr.n.u1Granularity));
4361 }
4362 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4363 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4364 {
4365 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4366 Assert(pCtx->ds.Attr.n.u1Present);
4367 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4368 Assert(!(pCtx->ds.Attr.u & 0xf00));
4369 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4370 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4371 || !(pCtx->ds.Attr.n.u1Granularity));
4372 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4373 || (pCtx->ds.Attr.n.u1Granularity));
4374 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4375 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4376 }
4377 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4378 {
4379 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4380 Assert(pCtx->es.Attr.n.u1Present);
4381 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4382 Assert(!(pCtx->es.Attr.u & 0xf00));
4383 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4384 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4385 || !(pCtx->es.Attr.n.u1Granularity));
4386 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4387 || (pCtx->es.Attr.n.u1Granularity));
4388 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4389 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4390 }
4391 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4392 {
4393 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4394 Assert(pCtx->fs.Attr.n.u1Present);
4395 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4396 Assert(!(pCtx->fs.Attr.u & 0xf00));
4397 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4398 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4399 || !(pCtx->fs.Attr.n.u1Granularity));
4400 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4401 || (pCtx->fs.Attr.n.u1Granularity));
4402 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4403 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4404 }
4405 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4406 {
4407 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4408 Assert(pCtx->gs.Attr.n.u1Present);
4409 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4410 Assert(!(pCtx->gs.Attr.u & 0xf00));
4411 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4412 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4413 || !(pCtx->gs.Attr.n.u1Granularity));
4414 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4415 || (pCtx->gs.Attr.n.u1Granularity));
4416 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4417 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4418 }
4419 /* 64-bit capable CPUs. */
4420# if HC_ARCH_BITS == 64
4421 Assert(!(pCtx->cs.u64Base >> 32));
4422 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4423 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4424 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4425# endif
4426 }
4427 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4428 || ( CPUMIsGuestInRealModeEx(pCtx)
4429 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4430 {
4431 /* Real and v86 mode checks. */
4432 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4433 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4434 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4435 {
4436 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4437 }
4438 else
4439 {
4440 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4441 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4442 }
4443
4444 /* CS */
4445 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4446 Assert(pCtx->cs.u32Limit == 0xffff);
4447 Assert(u32CSAttr == 0xf3);
4448 /* SS */
4449 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4450 Assert(pCtx->ss.u32Limit == 0xffff);
4451 Assert(u32SSAttr == 0xf3);
4452 /* DS */
4453 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4454 Assert(pCtx->ds.u32Limit == 0xffff);
4455 Assert(u32DSAttr == 0xf3);
4456 /* ES */
4457 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4458 Assert(pCtx->es.u32Limit == 0xffff);
4459 Assert(u32ESAttr == 0xf3);
4460 /* FS */
4461 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4462 Assert(pCtx->fs.u32Limit == 0xffff);
4463 Assert(u32FSAttr == 0xf3);
4464 /* GS */
4465 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4466 Assert(pCtx->gs.u32Limit == 0xffff);
4467 Assert(u32GSAttr == 0xf3);
4468 /* 64-bit capable CPUs. */
4469# if HC_ARCH_BITS == 64
4470 Assert(!(pCtx->cs.u64Base >> 32));
4471 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4472 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4473 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4474# endif
4475 }
4476}
4477#endif /* VBOX_STRICT */
4478
4479
4480/**
4481 * Writes a guest segment register into the guest-state area in the VMCS.
4482 *
4483 * @returns VBox status code.
4484 * @param pVCpu The cross context virtual CPU structure.
4485 * @param idxSel Index of the selector in the VMCS.
4486 * @param idxLimit Index of the segment limit in the VMCS.
4487 * @param idxBase Index of the segment base in the VMCS.
4488 * @param idxAccess Index of the access rights of the segment in the VMCS.
4489 * @param pSelReg Pointer to the segment selector.
4490 *
4491 * @remarks No-long-jump zone!!!
4492 */
4493static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4494 uint32_t idxAccess, PCPUMSELREG pSelReg)
4495{
4496 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4497 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4498 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4499 AssertRCReturn(rc, rc);
4500
4501 uint32_t u32Access = pSelReg->Attr.u;
4502 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4503 {
4504 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4505 u32Access = 0xf3;
4506 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4507 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4508 }
4509 else
4510 {
4511 /*
4512 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4513 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4514 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4515 * loaded in protected-mode have their attribute as 0.
4516 */
4517 if (!u32Access)
4518 u32Access = X86DESCATTR_UNUSABLE;
4519 }
4520
4521 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4522 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4523 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4524
4525 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4526 AssertRCReturn(rc, rc);
4527 return rc;
4528}
4529
4530
4531/**
4532 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4533 * into the guest-state area in the VMCS.
4534 *
4535 * @returns VBox status code.
4536 * @param pVCpu The cross context virtual CPU structure.
4537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4538 * out-of-sync. Make sure to update the required fields
4539 * before using them.
4540 *
4541 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4542 * @remarks No-long-jump zone!!!
4543 */
4544static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4545{
4546 int rc = VERR_INTERNAL_ERROR_5;
4547 PVM pVM = pVCpu->CTX_SUFF(pVM);
4548
4549 /*
4550 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4551 */
4552 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4553 {
4554 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4555 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4556 {
4557 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4558 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4559 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4560 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4561 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4562 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4563 }
4564
4565#ifdef VBOX_WITH_REM
4566 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4567 {
4568 Assert(pVM->hm.s.vmx.pRealModeTSS);
4569 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4570 if ( pVCpu->hm.s.vmx.fWasInRealMode
4571 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4572 {
4573 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4574 in real-mode (e.g. OpenBSD 4.0) */
4575 REMFlushTBs(pVM);
4576 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4577 pVCpu->hm.s.vmx.fWasInRealMode = false;
4578 }
4579 }
4580#endif
4581 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4582 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4583 AssertRCReturn(rc, rc);
4584 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4585 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4586 AssertRCReturn(rc, rc);
4587 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4588 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4589 AssertRCReturn(rc, rc);
4590 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4591 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4592 AssertRCReturn(rc, rc);
4593 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4594 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4595 AssertRCReturn(rc, rc);
4596 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4597 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4598 AssertRCReturn(rc, rc);
4599
4600#ifdef VBOX_STRICT
4601 /* Validate. */
4602 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4603#endif
4604
4605 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4606 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4607 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4608 }
4609
4610 /*
4611 * Guest TR.
4612 */
4613 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4614 {
4615 /*
4616 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4617 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4618 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4619 */
4620 uint16_t u16Sel = 0;
4621 uint32_t u32Limit = 0;
4622 uint64_t u64Base = 0;
4623 uint32_t u32AccessRights = 0;
4624
4625 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4626 {
4627 u16Sel = pMixedCtx->tr.Sel;
4628 u32Limit = pMixedCtx->tr.u32Limit;
4629 u64Base = pMixedCtx->tr.u64Base;
4630 u32AccessRights = pMixedCtx->tr.Attr.u;
4631 }
4632 else
4633 {
4634 Assert(pVM->hm.s.vmx.pRealModeTSS);
4635 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4636
4637 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4638 RTGCPHYS GCPhys;
4639 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4640 AssertRCReturn(rc, rc);
4641
4642 X86DESCATTR DescAttr;
4643 DescAttr.u = 0;
4644 DescAttr.n.u1Present = 1;
4645 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4646
4647 u16Sel = 0;
4648 u32Limit = HM_VTX_TSS_SIZE;
4649 u64Base = GCPhys; /* in real-mode phys = virt. */
4650 u32AccessRights = DescAttr.u;
4651 }
4652
4653 /* Validate. */
4654 Assert(!(u16Sel & RT_BIT(2)));
4655 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4656 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4657 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4658 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4659 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4660 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4661 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4662 Assert( (u32Limit & 0xfff) == 0xfff
4663 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4664 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4665 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4666
4667 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4668 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4669 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4670 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4671 AssertRCReturn(rc, rc);
4672
4673 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4674 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4675 }
4676
4677 /*
4678 * Guest GDTR.
4679 */
4680 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4681 {
4682 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4683 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4684 AssertRCReturn(rc, rc);
4685
4686 /* Validate. */
4687 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4688
4689 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4690 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4691 }
4692
4693 /*
4694 * Guest LDTR.
4695 */
4696 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4697 {
4698 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4699 uint32_t u32Access = 0;
4700 if (!pMixedCtx->ldtr.Attr.u)
4701 u32Access = X86DESCATTR_UNUSABLE;
4702 else
4703 u32Access = pMixedCtx->ldtr.Attr.u;
4704
4705 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4706 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4707 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4708 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4709 AssertRCReturn(rc, rc);
4710
4711 /* Validate. */
4712 if (!(u32Access & X86DESCATTR_UNUSABLE))
4713 {
4714 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4715 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4716 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4717 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4718 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4719 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4720 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4721 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4722 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4723 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4724 }
4725
4726 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4727 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4728 }
4729
4730 /*
4731 * Guest IDTR.
4732 */
4733 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4734 {
4735 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4736 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4737 AssertRCReturn(rc, rc);
4738
4739 /* Validate. */
4740 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4741
4742 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4743 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4744 }
4745
4746 return VINF_SUCCESS;
4747}
4748
4749
4750/**
4751 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4752 * areas.
4753 *
4754 * These MSRs will automatically be loaded to the host CPU on every successful
4755 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4756 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4757 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4758 *
4759 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4760 *
4761 * @returns VBox status code.
4762 * @param pVCpu The cross context virtual CPU structure.
4763 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4764 * out-of-sync. Make sure to update the required fields
4765 * before using them.
4766 *
4767 * @remarks No-long-jump zone!!!
4768 */
4769static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4770{
4771 AssertPtr(pVCpu);
4772 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4773
4774 /*
4775 * MSRs that we use the auto-load/store MSR area in the VMCS.
4776 */
4777 PVM pVM = pVCpu->CTX_SUFF(pVM);
4778 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4779 {
4780 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4781#if HC_ARCH_BITS == 32
4782 if (pVM->hm.s.fAllow64BitGuests)
4783 {
4784 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4785 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4786 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4787 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4788 AssertRCReturn(rc, rc);
4789# ifdef LOG_ENABLED
4790 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4791 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4792 {
4793 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4794 pMsr->u64Value));
4795 }
4796# endif
4797 }
4798#endif
4799 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4800 }
4801
4802 /*
4803 * Guest Sysenter MSRs.
4804 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4805 * VM-exits on WRMSRs for these MSRs.
4806 */
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4808 {
4809 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4810 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4811 }
4812
4813 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4814 {
4815 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4816 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4817 }
4818
4819 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4820 {
4821 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4822 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4823 }
4824
4825 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4826 {
4827 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4828 {
4829 /*
4830 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4831 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4832 */
4833 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4834 {
4835 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4836 AssertRCReturn(rc,rc);
4837 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4838 }
4839 else
4840 {
4841 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4842 NULL /* pfAddedAndUpdated */);
4843 AssertRCReturn(rc, rc);
4844
4845 /* We need to intercept reads too, see @bugref{7386#c16}. */
4846 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4847 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4848 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4849 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4850 }
4851 }
4852 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4853 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4854 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4855 }
4856
4857 return VINF_SUCCESS;
4858}
4859
4860
4861/**
4862 * Loads the guest activity state into the guest-state area in the VMCS.
4863 *
4864 * @returns VBox status code.
4865 * @param pVCpu The cross context virtual CPU structure.
4866 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4867 * out-of-sync. Make sure to update the required fields
4868 * before using them.
4869 *
4870 * @remarks No-long-jump zone!!!
4871 */
4872static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4873{
4874 NOREF(pMixedCtx);
4875 /** @todo See if we can make use of other states, e.g.
4876 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4877 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4878 {
4879 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4880 AssertRCReturn(rc, rc);
4881
4882 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4883 }
4884 return VINF_SUCCESS;
4885}
4886
4887
4888#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4889/**
4890 * Check if guest state allows safe use of 32-bit switcher again.
4891 *
4892 * Segment bases and protected mode structures must be 32-bit addressable
4893 * because the 32-bit switcher will ignore high dword when writing these VMCS
4894 * fields. See @bugref{8432} for details.
4895 *
4896 * @returns true if safe, false if must continue to use the 64-bit switcher.
4897 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4898 * out-of-sync. Make sure to update the required fields
4899 * before using them.
4900 *
4901 * @remarks No-long-jump zone!!!
4902 */
4903static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4904{
4905 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4906 return false;
4907 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4908 return false;
4909 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4910 return false;
4911 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4912 return false;
4913 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4914 return false;
4915 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4916 return false;
4917 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4918 return false;
4919 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4920 return false;
4921 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4922 return false;
4923 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4924 return false;
4925 /* All good, bases are 32-bit. */
4926 return true;
4927}
4928#endif
4929
4930
4931/**
4932 * Sets up the appropriate function to run guest code.
4933 *
4934 * @returns VBox status code.
4935 * @param pVCpu The cross context virtual CPU structure.
4936 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4937 * out-of-sync. Make sure to update the required fields
4938 * before using them.
4939 *
4940 * @remarks No-long-jump zone!!!
4941 */
4942static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4943{
4944 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4945 {
4946#ifndef VBOX_ENABLE_64_BITS_GUESTS
4947 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4948#endif
4949 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4950#if HC_ARCH_BITS == 32
4951 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4952 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4953 {
4954 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4955 {
4956 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4957 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4958 | HM_CHANGED_VMX_ENTRY_CTLS
4959 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4960 }
4961 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4962
4963 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4964 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4965 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4966 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4967 }
4968#else
4969 /* 64-bit host. */
4970 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4971#endif
4972 }
4973 else
4974 {
4975 /* Guest is not in long mode, use the 32-bit handler. */
4976#if HC_ARCH_BITS == 32
4977 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4978 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4979 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4980 {
4981 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4982 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4983 | HM_CHANGED_VMX_ENTRY_CTLS
4984 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4985 }
4986# ifdef VBOX_ENABLE_64_BITS_GUESTS
4987 /*
4988 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4989 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4990 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4991 * the much faster 32-bit switcher again.
4992 */
4993 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4994 {
4995 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4996 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4997 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4998 }
4999 else
5000 {
5001 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5002 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5003 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5004 {
5005 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5006 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5007 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5008 | HM_CHANGED_VMX_ENTRY_CTLS
5009 | HM_CHANGED_VMX_EXIT_CTLS
5010 | HM_CHANGED_HOST_CONTEXT);
5011 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5012 }
5013 }
5014# else
5015 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5016# endif
5017#else
5018 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5019#endif
5020 }
5021 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5022 return VINF_SUCCESS;
5023}
5024
5025
5026/**
5027 * Wrapper for running the guest code in VT-x.
5028 *
5029 * @returns VBox status code, no informational status codes.
5030 * @param pVM The cross context VM structure.
5031 * @param pVCpu The cross context virtual CPU structure.
5032 * @param pCtx Pointer to the guest-CPU context.
5033 *
5034 * @remarks No-long-jump zone!!!
5035 */
5036DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5037{
5038 /*
5039 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5040 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5041 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5042 */
5043 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5044 /** @todo Add stats for resume vs launch. */
5045#ifdef VBOX_WITH_KERNEL_USING_XMM
5046 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5047#else
5048 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5049#endif
5050 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5051 return rc;
5052}
5053
5054
5055/**
5056 * Reports world-switch error and dumps some useful debug info.
5057 *
5058 * @param pVM The cross context VM structure.
5059 * @param pVCpu The cross context virtual CPU structure.
5060 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5061 * @param pCtx Pointer to the guest-CPU context.
5062 * @param pVmxTransient Pointer to the VMX transient structure (only
5063 * exitReason updated).
5064 */
5065static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5066{
5067 Assert(pVM);
5068 Assert(pVCpu);
5069 Assert(pCtx);
5070 Assert(pVmxTransient);
5071 HMVMX_ASSERT_PREEMPT_SAFE();
5072
5073 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5074 switch (rcVMRun)
5075 {
5076 case VERR_VMX_INVALID_VMXON_PTR:
5077 AssertFailed();
5078 break;
5079 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5080 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5081 {
5082 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5083 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5084 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5085 AssertRC(rc);
5086
5087 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5088 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5089 Cannot do it here as we may have been long preempted. */
5090
5091#ifdef VBOX_STRICT
5092 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5093 pVmxTransient->uExitReason));
5094 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5095 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5096 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5097 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5098 else
5099 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5100 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5101 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5102
5103 /* VMX control bits. */
5104 uint32_t u32Val;
5105 uint64_t u64Val;
5106 RTHCUINTREG uHCReg;
5107 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5108 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5109 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5110 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5111 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5112 {
5113 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5114 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5115 }
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5120 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5121 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5124 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5125 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5126 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5127 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5128 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5129 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5130 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5131 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5132 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5133 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5134 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5135 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5136 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5137 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5138 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5139 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5140 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5141 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5143 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5144 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5145 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5146 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5147 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5148 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5149 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5150 if (pVM->hm.s.fNestedPaging)
5151 {
5152 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5153 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5154 }
5155
5156 /* Guest bits. */
5157 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5158 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5159 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5160 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5161 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5162 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5163 if (pVM->hm.s.vmx.fVpid)
5164 {
5165 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5166 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5167 }
5168
5169 /* Host bits. */
5170 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5171 Log4(("Host CR0 %#RHr\n", uHCReg));
5172 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5173 Log4(("Host CR3 %#RHr\n", uHCReg));
5174 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5175 Log4(("Host CR4 %#RHr\n", uHCReg));
5176
5177 RTGDTR HostGdtr;
5178 PCX86DESCHC pDesc;
5179 ASMGetGDTR(&HostGdtr);
5180 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5181 Log4(("Host CS %#08x\n", u32Val));
5182 if (u32Val < HostGdtr.cbGdt)
5183 {
5184 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5185 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5186 }
5187
5188 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5189 Log4(("Host DS %#08x\n", u32Val));
5190 if (u32Val < HostGdtr.cbGdt)
5191 {
5192 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5193 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5194 }
5195
5196 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5197 Log4(("Host ES %#08x\n", u32Val));
5198 if (u32Val < HostGdtr.cbGdt)
5199 {
5200 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5201 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5202 }
5203
5204 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5205 Log4(("Host FS %#08x\n", u32Val));
5206 if (u32Val < HostGdtr.cbGdt)
5207 {
5208 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5209 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5210 }
5211
5212 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5213 Log4(("Host GS %#08x\n", u32Val));
5214 if (u32Val < HostGdtr.cbGdt)
5215 {
5216 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5217 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5218 }
5219
5220 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5221 Log4(("Host SS %#08x\n", u32Val));
5222 if (u32Val < HostGdtr.cbGdt)
5223 {
5224 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5225 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5226 }
5227
5228 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5229 Log4(("Host TR %#08x\n", u32Val));
5230 if (u32Val < HostGdtr.cbGdt)
5231 {
5232 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5233 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5234 }
5235
5236 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5237 Log4(("Host TR Base %#RHv\n", uHCReg));
5238 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5239 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5240 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5241 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5242 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5243 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5244 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5245 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5246 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5247 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5248 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5249 Log4(("Host RSP %#RHv\n", uHCReg));
5250 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5251 Log4(("Host RIP %#RHv\n", uHCReg));
5252# if HC_ARCH_BITS == 64
5253 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5254 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5255 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5256 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5257 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5258 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5259# endif
5260#endif /* VBOX_STRICT */
5261 break;
5262 }
5263
5264 default:
5265 /* Impossible */
5266 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5267 break;
5268 }
5269 NOREF(pVM); NOREF(pCtx);
5270}
5271
5272
5273#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5274#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5275# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5276#endif
5277#ifdef VBOX_STRICT
5278static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5279{
5280 switch (idxField)
5281 {
5282 case VMX_VMCS_GUEST_RIP:
5283 case VMX_VMCS_GUEST_RSP:
5284 case VMX_VMCS_GUEST_SYSENTER_EIP:
5285 case VMX_VMCS_GUEST_SYSENTER_ESP:
5286 case VMX_VMCS_GUEST_GDTR_BASE:
5287 case VMX_VMCS_GUEST_IDTR_BASE:
5288 case VMX_VMCS_GUEST_CS_BASE:
5289 case VMX_VMCS_GUEST_DS_BASE:
5290 case VMX_VMCS_GUEST_ES_BASE:
5291 case VMX_VMCS_GUEST_FS_BASE:
5292 case VMX_VMCS_GUEST_GS_BASE:
5293 case VMX_VMCS_GUEST_SS_BASE:
5294 case VMX_VMCS_GUEST_LDTR_BASE:
5295 case VMX_VMCS_GUEST_TR_BASE:
5296 case VMX_VMCS_GUEST_CR3:
5297 return true;
5298 }
5299 return false;
5300}
5301
5302static bool hmR0VmxIsValidReadField(uint32_t idxField)
5303{
5304 switch (idxField)
5305 {
5306 /* Read-only fields. */
5307 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5308 return true;
5309 }
5310 /* Remaining readable fields should also be writable. */
5311 return hmR0VmxIsValidWriteField(idxField);
5312}
5313#endif /* VBOX_STRICT */
5314
5315
5316/**
5317 * Executes the specified handler in 64-bit mode.
5318 *
5319 * @returns VBox status code (no informational status codes).
5320 * @param pVM The cross context VM structure.
5321 * @param pVCpu The cross context virtual CPU structure.
5322 * @param pCtx Pointer to the guest CPU context.
5323 * @param enmOp The operation to perform.
5324 * @param cParams Number of parameters.
5325 * @param paParam Array of 32-bit parameters.
5326 */
5327VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5328 uint32_t cParams, uint32_t *paParam)
5329{
5330 NOREF(pCtx);
5331
5332 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5333 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5334 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5335 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5336
5337#ifdef VBOX_STRICT
5338 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5339 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5340
5341 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5342 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5343#endif
5344
5345 /* Disable interrupts. */
5346 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5347
5348#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5349 RTCPUID idHostCpu = RTMpCpuId();
5350 CPUMR0SetLApic(pVCpu, idHostCpu);
5351#endif
5352
5353 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5354 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5355
5356 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5357 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5358 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5359
5360 /* Leave VMX Root Mode. */
5361 VMXDisable();
5362
5363 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5364
5365 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5366 CPUMSetHyperEIP(pVCpu, enmOp);
5367 for (int i = (int)cParams - 1; i >= 0; i--)
5368 CPUMPushHyper(pVCpu, paParam[i]);
5369
5370 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5371
5372 /* Call the switcher. */
5373 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5374 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5375
5376 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5377 /* Make sure the VMX instructions don't cause #UD faults. */
5378 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5379
5380 /* Re-enter VMX Root Mode */
5381 int rc2 = VMXEnable(HCPhysCpuPage);
5382 if (RT_FAILURE(rc2))
5383 {
5384 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5385 ASMSetFlags(fOldEFlags);
5386 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5387 return rc2;
5388 }
5389
5390 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5391 AssertRC(rc2);
5392 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5393 Assert(!(ASMGetFlags() & X86_EFL_IF));
5394 ASMSetFlags(fOldEFlags);
5395 return rc;
5396}
5397
5398
5399/**
5400 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5401 * supporting 64-bit guests.
5402 *
5403 * @returns VBox status code.
5404 * @param fResume Whether to VMLAUNCH or VMRESUME.
5405 * @param pCtx Pointer to the guest-CPU context.
5406 * @param pCache Pointer to the VMCS cache.
5407 * @param pVM The cross context VM structure.
5408 * @param pVCpu The cross context virtual CPU structure.
5409 */
5410DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5411{
5412 NOREF(fResume);
5413
5414 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5415 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5416
5417#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5418 pCache->uPos = 1;
5419 pCache->interPD = PGMGetInterPaeCR3(pVM);
5420 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5421#endif
5422
5423#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5424 pCache->TestIn.HCPhysCpuPage = 0;
5425 pCache->TestIn.HCPhysVmcs = 0;
5426 pCache->TestIn.pCache = 0;
5427 pCache->TestOut.HCPhysVmcs = 0;
5428 pCache->TestOut.pCache = 0;
5429 pCache->TestOut.pCtx = 0;
5430 pCache->TestOut.eflags = 0;
5431#else
5432 NOREF(pCache);
5433#endif
5434
5435 uint32_t aParam[10];
5436 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5437 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5438 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5439 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5440 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5441 aParam[5] = 0;
5442 aParam[6] = VM_RC_ADDR(pVM, pVM);
5443 aParam[7] = 0;
5444 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5445 aParam[9] = 0;
5446
5447#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5448 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5449 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5450#endif
5451 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5452
5453#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5454 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5455 Assert(pCtx->dr[4] == 10);
5456 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5457#endif
5458
5459#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5460 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5461 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5462 pVCpu->hm.s.vmx.HCPhysVmcs));
5463 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5464 pCache->TestOut.HCPhysVmcs));
5465 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5466 pCache->TestOut.pCache));
5467 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5468 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5469 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5470 pCache->TestOut.pCtx));
5471 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5472#endif
5473 return rc;
5474}
5475
5476
5477/**
5478 * Initialize the VMCS-Read cache.
5479 *
5480 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5481 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5482 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5483 * (those that have a 32-bit FULL & HIGH part).
5484 *
5485 * @returns VBox status code.
5486 * @param pVM The cross context VM structure.
5487 * @param pVCpu The cross context virtual CPU structure.
5488 */
5489static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5490{
5491#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5492{ \
5493 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5494 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5495 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5496 ++cReadFields; \
5497}
5498
5499 AssertPtr(pVM);
5500 AssertPtr(pVCpu);
5501 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5502 uint32_t cReadFields = 0;
5503
5504 /*
5505 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5506 * and serve to indicate exceptions to the rules.
5507 */
5508
5509 /* Guest-natural selector base fields. */
5510#if 0
5511 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5514#endif
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5523 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5527#if 0
5528 /* Unused natural width guest-state fields. */
5529 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5531#endif
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5534
5535 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5536#if 0
5537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5540 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5541 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5542 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5543 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5544 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5545 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5546#endif
5547
5548 /* Natural width guest-state fields. */
5549 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5550#if 0
5551 /* Currently unused field. */
5552 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5553#endif
5554
5555 if (pVM->hm.s.fNestedPaging)
5556 {
5557 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5558 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5559 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5560 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5561 }
5562 else
5563 {
5564 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5565 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5566 }
5567
5568#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5569 return VINF_SUCCESS;
5570}
5571
5572
5573/**
5574 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5575 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5576 * darwin, running 64-bit guests).
5577 *
5578 * @returns VBox status code.
5579 * @param pVCpu The cross context virtual CPU structure.
5580 * @param idxField The VMCS field encoding.
5581 * @param u64Val 16, 32 or 64-bit value.
5582 */
5583VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5584{
5585 int rc;
5586 switch (idxField)
5587 {
5588 /*
5589 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5590 */
5591 /* 64-bit Control fields. */
5592 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5593 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5594 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5595 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5596 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5597 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5598 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5599 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5600 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5601 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5602 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5603 case VMX_VMCS64_CTRL_EPTP_FULL:
5604 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5605 /* 64-bit Guest-state fields. */
5606 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5607 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5608 case VMX_VMCS64_GUEST_PAT_FULL:
5609 case VMX_VMCS64_GUEST_EFER_FULL:
5610 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5611 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5612 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5613 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5614 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5615 /* 64-bit Host-state fields. */
5616 case VMX_VMCS64_HOST_PAT_FULL:
5617 case VMX_VMCS64_HOST_EFER_FULL:
5618 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5619 {
5620 rc = VMXWriteVmcs32(idxField, u64Val);
5621 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5622 break;
5623 }
5624
5625 /*
5626 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5627 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5628 */
5629 /* Natural-width Guest-state fields. */
5630 case VMX_VMCS_GUEST_CR3:
5631 case VMX_VMCS_GUEST_ES_BASE:
5632 case VMX_VMCS_GUEST_CS_BASE:
5633 case VMX_VMCS_GUEST_SS_BASE:
5634 case VMX_VMCS_GUEST_DS_BASE:
5635 case VMX_VMCS_GUEST_FS_BASE:
5636 case VMX_VMCS_GUEST_GS_BASE:
5637 case VMX_VMCS_GUEST_LDTR_BASE:
5638 case VMX_VMCS_GUEST_TR_BASE:
5639 case VMX_VMCS_GUEST_GDTR_BASE:
5640 case VMX_VMCS_GUEST_IDTR_BASE:
5641 case VMX_VMCS_GUEST_RSP:
5642 case VMX_VMCS_GUEST_RIP:
5643 case VMX_VMCS_GUEST_SYSENTER_ESP:
5644 case VMX_VMCS_GUEST_SYSENTER_EIP:
5645 {
5646 if (!(u64Val >> 32))
5647 {
5648 /* If this field is 64-bit, VT-x will zero out the top bits. */
5649 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5650 }
5651 else
5652 {
5653 /* Assert that only the 32->64 switcher case should ever come here. */
5654 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5655 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5656 }
5657 break;
5658 }
5659
5660 default:
5661 {
5662 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5663 rc = VERR_INVALID_PARAMETER;
5664 break;
5665 }
5666 }
5667 AssertRCReturn(rc, rc);
5668 return rc;
5669}
5670
5671
5672/**
5673 * Queue up a VMWRITE by using the VMCS write cache.
5674 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5675 *
5676 * @param pVCpu The cross context virtual CPU structure.
5677 * @param idxField The VMCS field encoding.
5678 * @param u64Val 16, 32 or 64-bit value.
5679 */
5680VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5681{
5682 AssertPtr(pVCpu);
5683 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5684
5685 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5686 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5687
5688 /* Make sure there are no duplicates. */
5689 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5690 {
5691 if (pCache->Write.aField[i] == idxField)
5692 {
5693 pCache->Write.aFieldVal[i] = u64Val;
5694 return VINF_SUCCESS;
5695 }
5696 }
5697
5698 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5699 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5700 pCache->Write.cValidEntries++;
5701 return VINF_SUCCESS;
5702}
5703#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5704
5705
5706/**
5707 * Sets up the usage of TSC-offsetting and updates the VMCS.
5708 *
5709 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5710 * VMX preemption timer.
5711 *
5712 * @returns VBox status code.
5713 * @param pVM The cross context VM structure.
5714 * @param pVCpu The cross context virtual CPU structure.
5715 *
5716 * @remarks No-long-jump zone!!!
5717 */
5718static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5719{
5720 int rc;
5721 bool fOffsettedTsc;
5722 bool fParavirtTsc;
5723 if (pVM->hm.s.vmx.fUsePreemptTimer)
5724 {
5725 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5726 &fOffsettedTsc, &fParavirtTsc);
5727
5728 /* Make sure the returned values have sane upper and lower boundaries. */
5729 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5730 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5731 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5732 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5733
5734 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5735 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5736 }
5737 else
5738 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5739
5740 /** @todo later optimize this to be done elsewhere and not before every
5741 * VM-entry. */
5742 if (fParavirtTsc)
5743 {
5744 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5745 information before every VM-entry, hence disable it for performance sake. */
5746#if 0
5747 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5748 AssertRC(rc);
5749#endif
5750 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5751 }
5752
5753 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5754 {
5755 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5756 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5757
5758 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5759 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5760 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5761 }
5762 else
5763 {
5764 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5765 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5766 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5767 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5768 }
5769}
5770
5771
5772/**
5773 * Determines if an exception is a contributory exception.
5774 *
5775 * Contributory exceptions are ones which can cause double-faults unless the
5776 * original exception was a benign exception. Page-fault is intentionally not
5777 * included here as it's a conditional contributory exception.
5778 *
5779 * @returns true if the exception is contributory, false otherwise.
5780 * @param uVector The exception vector.
5781 */
5782DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5783{
5784 switch (uVector)
5785 {
5786 case X86_XCPT_GP:
5787 case X86_XCPT_SS:
5788 case X86_XCPT_NP:
5789 case X86_XCPT_TS:
5790 case X86_XCPT_DE:
5791 return true;
5792 default:
5793 break;
5794 }
5795 return false;
5796}
5797
5798
5799/**
5800 * Sets an event as a pending event to be injected into the guest.
5801 *
5802 * @param pVCpu The cross context virtual CPU structure.
5803 * @param u32IntInfo The VM-entry interruption-information field.
5804 * @param cbInstr The VM-entry instruction length in bytes (for software
5805 * interrupts, exceptions and privileged software
5806 * exceptions).
5807 * @param u32ErrCode The VM-entry exception error code.
5808 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5809 * page-fault.
5810 *
5811 * @remarks Statistics counter assumes this is a guest event being injected or
5812 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5813 * always incremented.
5814 */
5815DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5816 RTGCUINTPTR GCPtrFaultAddress)
5817{
5818 Assert(!pVCpu->hm.s.Event.fPending);
5819 pVCpu->hm.s.Event.fPending = true;
5820 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5821 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5822 pVCpu->hm.s.Event.cbInstr = cbInstr;
5823 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5824}
5825
5826
5827/**
5828 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5829 *
5830 * @param pVCpu The cross context virtual CPU structure.
5831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5832 * out-of-sync. Make sure to update the required fields
5833 * before using them.
5834 */
5835DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5836{
5837 NOREF(pMixedCtx);
5838 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5839 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5840 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5841 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5842}
5843
5844
5845/**
5846 * Handle a condition that occurred while delivering an event through the guest
5847 * IDT.
5848 *
5849 * @returns Strict VBox status code (i.e. informational status codes too).
5850 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5851 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5852 * to continue execution of the guest which will delivery the \#DF.
5853 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5854 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5855 *
5856 * @param pVCpu The cross context virtual CPU structure.
5857 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5858 * out-of-sync. Make sure to update the required fields
5859 * before using them.
5860 * @param pVmxTransient Pointer to the VMX transient structure.
5861 *
5862 * @remarks No-long-jump zone!!!
5863 */
5864static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5865{
5866 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5867
5868 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5869 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5870
5871 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5872 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5873 {
5874 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5875 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5876
5877 typedef enum
5878 {
5879 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5880 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5881 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5882 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5883 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5884 } VMXREFLECTXCPT;
5885
5886 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5887 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5888 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5889 {
5890 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5891 {
5892 enmReflect = VMXREFLECTXCPT_XCPT;
5893#ifdef VBOX_STRICT
5894 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5895 && uExitVector == X86_XCPT_PF)
5896 {
5897 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5898 }
5899#endif
5900 if ( uExitVector == X86_XCPT_PF
5901 && uIdtVector == X86_XCPT_PF)
5902 {
5903 pVmxTransient->fVectoringDoublePF = true;
5904 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5905 }
5906 else if ( uExitVector == X86_XCPT_AC
5907 && uIdtVector == X86_XCPT_AC)
5908 {
5909 enmReflect = VMXREFLECTXCPT_HANG;
5910 Log4(("IDT: Nested #AC - Bad guest\n"));
5911 }
5912 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5913 && hmR0VmxIsContributoryXcpt(uExitVector)
5914 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5915 || uIdtVector == X86_XCPT_PF))
5916 {
5917 enmReflect = VMXREFLECTXCPT_DF;
5918 }
5919 else if (uIdtVector == X86_XCPT_DF)
5920 enmReflect = VMXREFLECTXCPT_TF;
5921 }
5922 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5923 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5924 {
5925 /*
5926 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5927 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5928 */
5929 enmReflect = VMXREFLECTXCPT_XCPT;
5930
5931 if (uExitVector == X86_XCPT_PF)
5932 {
5933 pVmxTransient->fVectoringPF = true;
5934 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5935 }
5936 }
5937 }
5938 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5939 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5940 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5941 {
5942 /*
5943 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5944 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5945 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5946 */
5947 enmReflect = VMXREFLECTXCPT_XCPT;
5948 }
5949
5950 /*
5951 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5952 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5953 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5954 *
5955 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5956 */
5957 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5958 && enmReflect == VMXREFLECTXCPT_XCPT
5959 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5960 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5961 {
5962 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5963 }
5964
5965 switch (enmReflect)
5966 {
5967 case VMXREFLECTXCPT_XCPT:
5968 {
5969 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5970 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5971 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5972
5973 uint32_t u32ErrCode = 0;
5974 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5975 {
5976 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5977 AssertRCReturn(rc2, rc2);
5978 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5979 }
5980
5981 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5982 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5983 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5984 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5985 rcStrict = VINF_SUCCESS;
5986 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5987 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5988
5989 break;
5990 }
5991
5992 case VMXREFLECTXCPT_DF:
5993 {
5994 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5995 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5996 rcStrict = VINF_HM_DOUBLE_FAULT;
5997 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5998 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5999
6000 break;
6001 }
6002
6003 case VMXREFLECTXCPT_TF:
6004 {
6005 rcStrict = VINF_EM_RESET;
6006 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6007 uExitVector));
6008 break;
6009 }
6010
6011 case VMXREFLECTXCPT_HANG:
6012 {
6013 rcStrict = VERR_EM_GUEST_CPU_HANG;
6014 break;
6015 }
6016
6017 default:
6018 Assert(rcStrict == VINF_SUCCESS);
6019 break;
6020 }
6021 }
6022 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6023 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6024 && uExitVector != X86_XCPT_DF
6025 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6026 {
6027 /*
6028 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6029 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6030 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6031 */
6032 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6033 {
6034 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6035 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6036 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6037 }
6038 }
6039
6040 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6041 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6042 return rcStrict;
6043}
6044
6045
6046/**
6047 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6048 *
6049 * @returns VBox status code.
6050 * @param pVCpu The cross context virtual CPU structure.
6051 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6052 * out-of-sync. Make sure to update the required fields
6053 * before using them.
6054 *
6055 * @remarks No-long-jump zone!!!
6056 */
6057static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6058{
6059 NOREF(pMixedCtx);
6060
6061 /*
6062 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6063 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6064 */
6065 VMMRZCallRing3Disable(pVCpu);
6066 HM_DISABLE_PREEMPT();
6067
6068 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6069 {
6070 uint32_t uVal = 0;
6071 uint32_t uShadow = 0;
6072 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6073 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6074 AssertRCReturn(rc, rc);
6075
6076 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6077 CPUMSetGuestCR0(pVCpu, uVal);
6078 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6079 }
6080
6081 HM_RESTORE_PREEMPT();
6082 VMMRZCallRing3Enable(pVCpu);
6083 return VINF_SUCCESS;
6084}
6085
6086
6087/**
6088 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6089 *
6090 * @returns VBox status code.
6091 * @param pVCpu The cross context virtual CPU structure.
6092 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6093 * out-of-sync. Make sure to update the required fields
6094 * before using them.
6095 *
6096 * @remarks No-long-jump zone!!!
6097 */
6098static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099{
6100 NOREF(pMixedCtx);
6101
6102 int rc = VINF_SUCCESS;
6103 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6104 {
6105 uint32_t uVal = 0;
6106 uint32_t uShadow = 0;
6107 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6108 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6109 AssertRCReturn(rc, rc);
6110
6111 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6112 CPUMSetGuestCR4(pVCpu, uVal);
6113 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6114 }
6115 return rc;
6116}
6117
6118
6119/**
6120 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6121 *
6122 * @returns VBox status code.
6123 * @param pVCpu The cross context virtual CPU structure.
6124 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6125 * out-of-sync. Make sure to update the required fields
6126 * before using them.
6127 *
6128 * @remarks No-long-jump zone!!!
6129 */
6130static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6131{
6132 int rc = VINF_SUCCESS;
6133 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6134 {
6135 uint64_t u64Val = 0;
6136 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6137 AssertRCReturn(rc, rc);
6138
6139 pMixedCtx->rip = u64Val;
6140 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6141 }
6142 return rc;
6143}
6144
6145
6146/**
6147 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6148 *
6149 * @returns VBox status code.
6150 * @param pVCpu The cross context virtual CPU structure.
6151 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6152 * out-of-sync. Make sure to update the required fields
6153 * before using them.
6154 *
6155 * @remarks No-long-jump zone!!!
6156 */
6157static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6158{
6159 int rc = VINF_SUCCESS;
6160 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6161 {
6162 uint64_t u64Val = 0;
6163 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6164 AssertRCReturn(rc, rc);
6165
6166 pMixedCtx->rsp = u64Val;
6167 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6168 }
6169 return rc;
6170}
6171
6172
6173/**
6174 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6175 *
6176 * @returns VBox status code.
6177 * @param pVCpu The cross context virtual CPU structure.
6178 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6179 * out-of-sync. Make sure to update the required fields
6180 * before using them.
6181 *
6182 * @remarks No-long-jump zone!!!
6183 */
6184static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6185{
6186 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6187 {
6188 uint32_t uVal = 0;
6189 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6190 AssertRCReturn(rc, rc);
6191
6192 pMixedCtx->eflags.u32 = uVal;
6193 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6194 {
6195 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6196 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6197
6198 pMixedCtx->eflags.Bits.u1VM = 0;
6199 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6200 }
6201
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6203 }
6204 return VINF_SUCCESS;
6205}
6206
6207
6208/**
6209 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6210 * guest-CPU context.
6211 */
6212DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6213{
6214 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6215 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6216 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6217 return rc;
6218}
6219
6220
6221/**
6222 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6223 * from the guest-state area in the VMCS.
6224 *
6225 * @param pVCpu The cross context virtual CPU structure.
6226 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6227 * out-of-sync. Make sure to update the required fields
6228 * before using them.
6229 *
6230 * @remarks No-long-jump zone!!!
6231 */
6232static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6233{
6234 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6235 {
6236 uint32_t uIntrState = 0;
6237 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6238 AssertRC(rc);
6239
6240 if (!uIntrState)
6241 {
6242 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6243 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6244
6245 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6246 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6247 }
6248 else
6249 {
6250 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6251 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6252 {
6253 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6254 AssertRC(rc);
6255 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6256 AssertRC(rc);
6257
6258 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6259 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6260 }
6261 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6262 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6263
6264 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6265 {
6266 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6267 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6268 }
6269 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6270 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6271 }
6272
6273 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6274 }
6275}
6276
6277
6278/**
6279 * Saves the guest's activity state.
6280 *
6281 * @returns VBox status code.
6282 * @param pVCpu The cross context virtual CPU structure.
6283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6284 * out-of-sync. Make sure to update the required fields
6285 * before using them.
6286 *
6287 * @remarks No-long-jump zone!!!
6288 */
6289static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6290{
6291 NOREF(pMixedCtx);
6292 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6293 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6294 return VINF_SUCCESS;
6295}
6296
6297
6298/**
6299 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6300 * the current VMCS into the guest-CPU context.
6301 *
6302 * @returns VBox status code.
6303 * @param pVCpu The cross context virtual CPU structure.
6304 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6305 * out-of-sync. Make sure to update the required fields
6306 * before using them.
6307 *
6308 * @remarks No-long-jump zone!!!
6309 */
6310static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6311{
6312 int rc = VINF_SUCCESS;
6313 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6314 {
6315 uint32_t u32Val = 0;
6316 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6317 pMixedCtx->SysEnter.cs = u32Val;
6318 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6319 }
6320
6321 uint64_t u64Val = 0;
6322 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6323 {
6324 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6325 pMixedCtx->SysEnter.eip = u64Val;
6326 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6327 }
6328 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6329 {
6330 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6331 pMixedCtx->SysEnter.esp = u64Val;
6332 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6333 }
6334 return rc;
6335}
6336
6337
6338/**
6339 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6340 * the CPU back into the guest-CPU context.
6341 *
6342 * @returns VBox status code.
6343 * @param pVCpu The cross context virtual CPU structure.
6344 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6345 * out-of-sync. Make sure to update the required fields
6346 * before using them.
6347 *
6348 * @remarks No-long-jump zone!!!
6349 */
6350static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6351{
6352 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6353 VMMRZCallRing3Disable(pVCpu);
6354 HM_DISABLE_PREEMPT();
6355
6356 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6357 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6358 {
6359 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6360 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6361 }
6362
6363 HM_RESTORE_PREEMPT();
6364 VMMRZCallRing3Enable(pVCpu);
6365
6366 return VINF_SUCCESS;
6367}
6368
6369
6370/**
6371 * Saves the auto load/store'd guest MSRs from the current VMCS into
6372 * the guest-CPU context.
6373 *
6374 * @returns VBox status code.
6375 * @param pVCpu The cross context virtual CPU structure.
6376 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6377 * out-of-sync. Make sure to update the required fields
6378 * before using them.
6379 *
6380 * @remarks No-long-jump zone!!!
6381 */
6382static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6383{
6384 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6385 return VINF_SUCCESS;
6386
6387 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6388 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6389 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6390 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6391 {
6392 switch (pMsr->u32Msr)
6393 {
6394 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6395 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6396 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6397 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6398 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6399 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6400 break;
6401
6402 default:
6403 {
6404 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6405 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6406 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6407 }
6408 }
6409 }
6410
6411 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6412 return VINF_SUCCESS;
6413}
6414
6415
6416/**
6417 * Saves the guest control registers from the current VMCS into the guest-CPU
6418 * context.
6419 *
6420 * @returns VBox status code.
6421 * @param pVCpu The cross context virtual CPU structure.
6422 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6423 * out-of-sync. Make sure to update the required fields
6424 * before using them.
6425 *
6426 * @remarks No-long-jump zone!!!
6427 */
6428static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6429{
6430 /* Guest CR0. Guest FPU. */
6431 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6432 AssertRCReturn(rc, rc);
6433
6434 /* Guest CR4. */
6435 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6436 AssertRCReturn(rc, rc);
6437
6438 /* Guest CR2 - updated always during the world-switch or in #PF. */
6439 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6440 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6441 {
6442 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6443 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6444
6445 PVM pVM = pVCpu->CTX_SUFF(pVM);
6446 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6447 || ( pVM->hm.s.fNestedPaging
6448 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6449 {
6450 uint64_t u64Val = 0;
6451 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6452 if (pMixedCtx->cr3 != u64Val)
6453 {
6454 CPUMSetGuestCR3(pVCpu, u64Val);
6455 if (VMMRZCallRing3IsEnabled(pVCpu))
6456 {
6457 PGMUpdateCR3(pVCpu, u64Val);
6458 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6459 }
6460 else
6461 {
6462 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6463 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6464 }
6465 }
6466
6467 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6468 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6469 {
6470 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6471 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6472 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6473 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6474 AssertRCReturn(rc, rc);
6475
6476 if (VMMRZCallRing3IsEnabled(pVCpu))
6477 {
6478 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6479 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6480 }
6481 else
6482 {
6483 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6484 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6485 }
6486 }
6487 }
6488
6489 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6490 }
6491
6492 /*
6493 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6494 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6495 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6496 *
6497 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6498 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6499 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6500 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6501 *
6502 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6503 */
6504 if (VMMRZCallRing3IsEnabled(pVCpu))
6505 {
6506 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6507 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6508
6509 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6510 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6511
6512 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6513 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6514 }
6515
6516 return rc;
6517}
6518
6519
6520/**
6521 * Reads a guest segment register from the current VMCS into the guest-CPU
6522 * context.
6523 *
6524 * @returns VBox status code.
6525 * @param pVCpu The cross context virtual CPU structure.
6526 * @param idxSel Index of the selector in the VMCS.
6527 * @param idxLimit Index of the segment limit in the VMCS.
6528 * @param idxBase Index of the segment base in the VMCS.
6529 * @param idxAccess Index of the access rights of the segment in the VMCS.
6530 * @param pSelReg Pointer to the segment selector.
6531 *
6532 * @remarks No-long-jump zone!!!
6533 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6534 * macro as that takes care of whether to read from the VMCS cache or
6535 * not.
6536 */
6537DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6538 PCPUMSELREG pSelReg)
6539{
6540 NOREF(pVCpu);
6541
6542 uint32_t u32Val = 0;
6543 int rc = VMXReadVmcs32(idxSel, &u32Val);
6544 AssertRCReturn(rc, rc);
6545 pSelReg->Sel = (uint16_t)u32Val;
6546 pSelReg->ValidSel = (uint16_t)u32Val;
6547 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6548
6549 rc = VMXReadVmcs32(idxLimit, &u32Val);
6550 AssertRCReturn(rc, rc);
6551 pSelReg->u32Limit = u32Val;
6552
6553 uint64_t u64Val = 0;
6554 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6555 AssertRCReturn(rc, rc);
6556 pSelReg->u64Base = u64Val;
6557
6558 rc = VMXReadVmcs32(idxAccess, &u32Val);
6559 AssertRCReturn(rc, rc);
6560 pSelReg->Attr.u = u32Val;
6561
6562 /*
6563 * If VT-x marks the segment as unusable, most other bits remain undefined:
6564 * - For CS the L, D and G bits have meaning.
6565 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6566 * - For the remaining data segments no bits are defined.
6567 *
6568 * The present bit and the unusable bit has been observed to be set at the
6569 * same time (the selector was supposed to be invalid as we started executing
6570 * a V8086 interrupt in ring-0).
6571 *
6572 * What should be important for the rest of the VBox code, is that the P bit is
6573 * cleared. Some of the other VBox code recognizes the unusable bit, but
6574 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6575 * safe side here, we'll strip off P and other bits we don't care about. If
6576 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6577 *
6578 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6579 */
6580 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6581 {
6582 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6583
6584 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6585 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6586 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6587
6588 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6589#ifdef DEBUG_bird
6590 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6591 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6592 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6593#endif
6594 }
6595 return VINF_SUCCESS;
6596}
6597
6598
6599#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6600# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6601 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6602 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6603#else
6604# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6605 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6606 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6607#endif
6608
6609
6610/**
6611 * Saves the guest segment registers from the current VMCS into the guest-CPU
6612 * context.
6613 *
6614 * @returns VBox status code.
6615 * @param pVCpu The cross context virtual CPU structure.
6616 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6617 * out-of-sync. Make sure to update the required fields
6618 * before using them.
6619 *
6620 * @remarks No-long-jump zone!!!
6621 */
6622static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6623{
6624 /* Guest segment registers. */
6625 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6626 {
6627 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6628 AssertRCReturn(rc, rc);
6629
6630 rc = VMXLOCAL_READ_SEG(CS, cs);
6631 rc |= VMXLOCAL_READ_SEG(SS, ss);
6632 rc |= VMXLOCAL_READ_SEG(DS, ds);
6633 rc |= VMXLOCAL_READ_SEG(ES, es);
6634 rc |= VMXLOCAL_READ_SEG(FS, fs);
6635 rc |= VMXLOCAL_READ_SEG(GS, gs);
6636 AssertRCReturn(rc, rc);
6637
6638 /* Restore segment attributes for real-on-v86 mode hack. */
6639 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6640 {
6641 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6642 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6643 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6644 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6645 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6646 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6647 }
6648 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6649 }
6650
6651 return VINF_SUCCESS;
6652}
6653
6654
6655/**
6656 * Saves the guest descriptor table registers and task register from the current
6657 * VMCS into the guest-CPU context.
6658 *
6659 * @returns VBox status code.
6660 * @param pVCpu The cross context virtual CPU structure.
6661 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6662 * out-of-sync. Make sure to update the required fields
6663 * before using them.
6664 *
6665 * @remarks No-long-jump zone!!!
6666 */
6667static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6668{
6669 int rc = VINF_SUCCESS;
6670
6671 /* Guest LDTR. */
6672 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6673 {
6674 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6675 AssertRCReturn(rc, rc);
6676 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6677 }
6678
6679 /* Guest GDTR. */
6680 uint64_t u64Val = 0;
6681 uint32_t u32Val = 0;
6682 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6683 {
6684 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6685 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6686 pMixedCtx->gdtr.pGdt = u64Val;
6687 pMixedCtx->gdtr.cbGdt = u32Val;
6688 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6689 }
6690
6691 /* Guest IDTR. */
6692 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6693 {
6694 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6695 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6696 pMixedCtx->idtr.pIdt = u64Val;
6697 pMixedCtx->idtr.cbIdt = u32Val;
6698 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6699 }
6700
6701 /* Guest TR. */
6702 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6703 {
6704 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6705 AssertRCReturn(rc, rc);
6706
6707 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6708 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6709 {
6710 rc = VMXLOCAL_READ_SEG(TR, tr);
6711 AssertRCReturn(rc, rc);
6712 }
6713 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6714 }
6715 return rc;
6716}
6717
6718#undef VMXLOCAL_READ_SEG
6719
6720
6721/**
6722 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6723 * context.
6724 *
6725 * @returns VBox status code.
6726 * @param pVCpu The cross context virtual CPU structure.
6727 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6728 * out-of-sync. Make sure to update the required fields
6729 * before using them.
6730 *
6731 * @remarks No-long-jump zone!!!
6732 */
6733static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6734{
6735 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6736 {
6737 if (!pVCpu->hm.s.fUsingHyperDR7)
6738 {
6739 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6740 uint32_t u32Val;
6741 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6742 pMixedCtx->dr[7] = u32Val;
6743 }
6744
6745 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6746 }
6747 return VINF_SUCCESS;
6748}
6749
6750
6751/**
6752 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6753 *
6754 * @returns VBox status code.
6755 * @param pVCpu The cross context virtual CPU structure.
6756 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6757 * out-of-sync. Make sure to update the required fields
6758 * before using them.
6759 *
6760 * @remarks No-long-jump zone!!!
6761 */
6762static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6763{
6764 NOREF(pMixedCtx);
6765
6766 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6767 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6768 return VINF_SUCCESS;
6769}
6770
6771
6772/**
6773 * Saves the entire guest state from the currently active VMCS into the
6774 * guest-CPU context.
6775 *
6776 * This essentially VMREADs all guest-data.
6777 *
6778 * @returns VBox status code.
6779 * @param pVCpu The cross context virtual CPU structure.
6780 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6781 * out-of-sync. Make sure to update the required fields
6782 * before using them.
6783 */
6784static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6785{
6786 Assert(pVCpu);
6787 Assert(pMixedCtx);
6788
6789 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6790 return VINF_SUCCESS;
6791
6792 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6793 again on the ring-3 callback path, there is no real need to. */
6794 if (VMMRZCallRing3IsEnabled(pVCpu))
6795 VMMR0LogFlushDisable(pVCpu);
6796 else
6797 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6798 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6799
6800 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6801 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6802
6803 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6804 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6805
6806 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6807 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6808
6809 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6810 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6811
6812 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6814
6815 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6817
6818 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6820
6821 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6823
6824 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6826
6827 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6829
6830 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6831 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6832 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6833
6834 if (VMMRZCallRing3IsEnabled(pVCpu))
6835 VMMR0LogFlushEnable(pVCpu);
6836
6837 return VINF_SUCCESS;
6838}
6839
6840
6841/**
6842 * Saves basic guest registers needed for IEM instruction execution.
6843 *
6844 * @returns VBox status code (OR-able).
6845 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6846 * @param pMixedCtx Pointer to the CPU context of the guest.
6847 * @param fMemory Whether the instruction being executed operates on
6848 * memory or not. Only CR0 is synced up if clear.
6849 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6850 */
6851static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6852{
6853 /*
6854 * We assume all general purpose registers other than RSP are available.
6855 *
6856 * - RIP is a must, as it will be incremented or otherwise changed.
6857 * - RFLAGS are always required to figure the CPL.
6858 * - RSP isn't always required, however it's a GPR, so frequently required.
6859 * - SS and CS are the only segment register needed if IEM doesn't do memory
6860 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6861 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6862 * be required for memory accesses.
6863 *
6864 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6865 */
6866 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6867 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6868 if (fNeedRsp)
6869 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6870 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6871 if (!fMemory)
6872 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6873 else
6874 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6875 AssertRCReturn(rc, rc);
6876 return rc;
6877}
6878
6879
6880/**
6881 * Ensures that we've got a complete basic guest-context.
6882 *
6883 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6884 * is for the interpreter.
6885 *
6886 * @returns VBox status code.
6887 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6888 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6889 * needing to be synced in.
6890 * @thread EMT(pVCpu)
6891 */
6892VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6893{
6894 /* Note! Since this is only applicable to VT-x, the implementation is placed
6895 in the VT-x part of the sources instead of the generic stuff. */
6896 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6897 {
6898 /* For now, imply that the caller might change everything too. */
6899 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6900 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6901 }
6902 return VINF_SUCCESS;
6903}
6904
6905
6906/**
6907 * Check per-VM and per-VCPU force flag actions that require us to go back to
6908 * ring-3 for one reason or another.
6909 *
6910 * @returns Strict VBox status code (i.e. informational status codes too)
6911 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6912 * ring-3.
6913 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6914 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6915 * interrupts)
6916 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6917 * all EMTs to be in ring-3.
6918 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6919 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6920 * to the EM loop.
6921 *
6922 * @param pVM The cross context VM structure.
6923 * @param pVCpu The cross context virtual CPU structure.
6924 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6925 * out-of-sync. Make sure to update the required fields
6926 * before using them.
6927 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6928 */
6929static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6930{
6931 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6932
6933 /*
6934 * Anything pending? Should be more likely than not if we're doing a good job.
6935 */
6936 if ( !fStepping
6937 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6938 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6939 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6940 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6941 return VINF_SUCCESS;
6942
6943 /* We need the control registers now, make sure the guest-CPU context is updated. */
6944 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6945 AssertRCReturn(rc3, rc3);
6946
6947 /* Pending HM CR3 sync. */
6948 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6949 {
6950 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6951 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6952 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6953 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6954 }
6955
6956 /* Pending HM PAE PDPEs. */
6957 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6958 {
6959 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6960 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6961 }
6962
6963 /* Pending PGM C3 sync. */
6964 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6965 {
6966 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6967 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6968 if (rcStrict2 != VINF_SUCCESS)
6969 {
6970 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6971 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6972 return rcStrict2;
6973 }
6974 }
6975
6976 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6977 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6978 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6979 {
6980 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6981 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6982 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6983 return rc2;
6984 }
6985
6986 /* Pending VM request packets, such as hardware interrupts. */
6987 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6988 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6989 {
6990 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6991 return VINF_EM_PENDING_REQUEST;
6992 }
6993
6994 /* Pending PGM pool flushes. */
6995 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6996 {
6997 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6998 return VINF_PGM_POOL_FLUSH_PENDING;
6999 }
7000
7001 /* Pending DMA requests. */
7002 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7003 {
7004 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7005 return VINF_EM_RAW_TO_R3;
7006 }
7007
7008 return VINF_SUCCESS;
7009}
7010
7011
7012/**
7013 * Converts any TRPM trap into a pending HM event. This is typically used when
7014 * entering from ring-3 (not longjmp returns).
7015 *
7016 * @param pVCpu The cross context virtual CPU structure.
7017 */
7018static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7019{
7020 Assert(TRPMHasTrap(pVCpu));
7021 Assert(!pVCpu->hm.s.Event.fPending);
7022
7023 uint8_t uVector;
7024 TRPMEVENT enmTrpmEvent;
7025 RTGCUINT uErrCode;
7026 RTGCUINTPTR GCPtrFaultAddress;
7027 uint8_t cbInstr;
7028
7029 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7030 AssertRC(rc);
7031
7032 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7033 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7034 if (enmTrpmEvent == TRPM_TRAP)
7035 {
7036 switch (uVector)
7037 {
7038 case X86_XCPT_NMI:
7039 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7040 break;
7041
7042 case X86_XCPT_BP:
7043 case X86_XCPT_OF:
7044 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7045 break;
7046
7047 case X86_XCPT_PF:
7048 case X86_XCPT_DF:
7049 case X86_XCPT_TS:
7050 case X86_XCPT_NP:
7051 case X86_XCPT_SS:
7052 case X86_XCPT_GP:
7053 case X86_XCPT_AC:
7054 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7055 /* no break! */
7056 default:
7057 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7058 break;
7059 }
7060 }
7061 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7062 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7063 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7064 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7065 else
7066 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7067
7068 rc = TRPMResetTrap(pVCpu);
7069 AssertRC(rc);
7070 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7071 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7072
7073 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7074}
7075
7076
7077/**
7078 * Converts the pending HM event into a TRPM trap.
7079 *
7080 * @param pVCpu The cross context virtual CPU structure.
7081 */
7082static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7083{
7084 Assert(pVCpu->hm.s.Event.fPending);
7085
7086 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7087 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7088 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7089 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7090
7091 /* If a trap was already pending, we did something wrong! */
7092 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7093
7094 TRPMEVENT enmTrapType;
7095 switch (uVectorType)
7096 {
7097 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7098 enmTrapType = TRPM_HARDWARE_INT;
7099 break;
7100
7101 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7102 enmTrapType = TRPM_SOFTWARE_INT;
7103 break;
7104
7105 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7106 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7107 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7108 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7109 enmTrapType = TRPM_TRAP;
7110 break;
7111
7112 default:
7113 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7114 enmTrapType = TRPM_32BIT_HACK;
7115 break;
7116 }
7117
7118 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7119
7120 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7121 AssertRC(rc);
7122
7123 if (fErrorCodeValid)
7124 TRPMSetErrorCode(pVCpu, uErrorCode);
7125
7126 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7127 && uVector == X86_XCPT_PF)
7128 {
7129 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7130 }
7131 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7132 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7133 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7134 {
7135 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7136 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7137 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7138 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7139 }
7140
7141 /* Clear any pending events from the VMCS. */
7142 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7143 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7144
7145 /* We're now done converting the pending event. */
7146 pVCpu->hm.s.Event.fPending = false;
7147}
7148
7149
7150/**
7151 * Does the necessary state syncing before returning to ring-3 for any reason
7152 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7153 *
7154 * @returns VBox status code.
7155 * @param pVCpu The cross context virtual CPU structure.
7156 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7157 * be out-of-sync. Make sure to update the required
7158 * fields before using them.
7159 * @param fSaveGuestState Whether to save the guest state or not.
7160 *
7161 * @remarks No-long-jmp zone!!!
7162 */
7163static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7164{
7165 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7166 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7167
7168 RTCPUID idCpu = RTMpCpuId();
7169 Log4Func(("HostCpuId=%u\n", idCpu));
7170
7171 /*
7172 * !!! IMPORTANT !!!
7173 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7174 */
7175
7176 /* Save the guest state if necessary. */
7177 if ( fSaveGuestState
7178 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7179 {
7180 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7181 AssertRCReturn(rc, rc);
7182 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7183 }
7184
7185 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7186 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7187 {
7188 if (fSaveGuestState)
7189 {
7190 /* We shouldn't reload CR0 without saving it first. */
7191 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7192 AssertRCReturn(rc, rc);
7193 }
7194 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7195 }
7196
7197 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7198#ifdef VBOX_STRICT
7199 if (CPUMIsHyperDebugStateActive(pVCpu))
7200 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7201#endif
7202 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7203 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7204 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7205 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7206
7207#if HC_ARCH_BITS == 64
7208 /* Restore host-state bits that VT-x only restores partially. */
7209 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7210 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7211 {
7212 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7213 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7214 }
7215 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7216#endif
7217
7218 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7219 if (pVCpu->hm.s.vmx.fLazyMsrs)
7220 {
7221 /* We shouldn't reload the guest MSRs without saving it first. */
7222 if (!fSaveGuestState)
7223 {
7224 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7225 AssertRCReturn(rc, rc);
7226 }
7227 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7228 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7229 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7230 }
7231
7232 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7233 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7234
7235 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7236 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7237 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7238 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7239 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7240 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7241 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7242 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7243
7244 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7245
7246 /** @todo This partially defeats the purpose of having preemption hooks.
7247 * The problem is, deregistering the hooks should be moved to a place that
7248 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7249 * context.
7250 */
7251 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7252 {
7253 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7254 AssertRCReturn(rc, rc);
7255
7256 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7257 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7258 }
7259 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7260 NOREF(idCpu);
7261
7262 return VINF_SUCCESS;
7263}
7264
7265
7266/**
7267 * Leaves the VT-x session.
7268 *
7269 * @returns VBox status code.
7270 * @param pVCpu The cross context virtual CPU structure.
7271 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7272 * out-of-sync. Make sure to update the required fields
7273 * before using them.
7274 *
7275 * @remarks No-long-jmp zone!!!
7276 */
7277DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7278{
7279 HM_DISABLE_PREEMPT();
7280 HMVMX_ASSERT_CPU_SAFE();
7281 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7282 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7283
7284 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7285 and done this from the VMXR0ThreadCtxCallback(). */
7286 if (!pVCpu->hm.s.fLeaveDone)
7287 {
7288 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7289 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7290 pVCpu->hm.s.fLeaveDone = true;
7291 }
7292 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7293
7294 /*
7295 * !!! IMPORTANT !!!
7296 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7297 */
7298
7299 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7300 /** @todo Deregistering here means we need to VMCLEAR always
7301 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7302 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7303 VMMR0ThreadCtxHookDisable(pVCpu);
7304
7305 /* Leave HM context. This takes care of local init (term). */
7306 int rc = HMR0LeaveCpu(pVCpu);
7307
7308 HM_RESTORE_PREEMPT();
7309 return rc;
7310}
7311
7312
7313/**
7314 * Does the necessary state syncing before doing a longjmp to ring-3.
7315 *
7316 * @returns VBox status code.
7317 * @param pVCpu The cross context virtual CPU structure.
7318 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7319 * out-of-sync. Make sure to update the required fields
7320 * before using them.
7321 *
7322 * @remarks No-long-jmp zone!!!
7323 */
7324DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7325{
7326 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7327}
7328
7329
7330/**
7331 * Take necessary actions before going back to ring-3.
7332 *
7333 * An action requires us to go back to ring-3. This function does the necessary
7334 * steps before we can safely return to ring-3. This is not the same as longjmps
7335 * to ring-3, this is voluntary and prepares the guest so it may continue
7336 * executing outside HM (recompiler/IEM).
7337 *
7338 * @returns VBox status code.
7339 * @param pVM The cross context VM structure.
7340 * @param pVCpu The cross context virtual CPU structure.
7341 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7342 * out-of-sync. Make sure to update the required fields
7343 * before using them.
7344 * @param rcExit The reason for exiting to ring-3. Can be
7345 * VINF_VMM_UNKNOWN_RING3_CALL.
7346 */
7347static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7348{
7349 Assert(pVM);
7350 Assert(pVCpu);
7351 Assert(pMixedCtx);
7352 HMVMX_ASSERT_PREEMPT_SAFE();
7353
7354 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7355 {
7356 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7357 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7358 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7359 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7360 }
7361
7362 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7363 VMMRZCallRing3Disable(pVCpu);
7364 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7365
7366 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7367 if (pVCpu->hm.s.Event.fPending)
7368 {
7369 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7370 Assert(!pVCpu->hm.s.Event.fPending);
7371 }
7372
7373 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7374 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7375
7376 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7377 and if we're injecting an event we should have a TRPM trap pending. */
7378 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7379#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7380 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7381#endif
7382
7383 /* Save guest state and restore host state bits. */
7384 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7385 AssertRCReturn(rc, rc);
7386 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7387 /* Thread-context hooks are unregistered at this point!!! */
7388
7389 /* Sync recompiler state. */
7390 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7391 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7392 | CPUM_CHANGED_LDTR
7393 | CPUM_CHANGED_GDTR
7394 | CPUM_CHANGED_IDTR
7395 | CPUM_CHANGED_TR
7396 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7397 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7398 if ( pVM->hm.s.fNestedPaging
7399 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7400 {
7401 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7402 }
7403
7404 Assert(!pVCpu->hm.s.fClearTrapFlag);
7405
7406 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7407 if (rcExit != VINF_EM_RAW_INTERRUPT)
7408 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7409
7410 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7411
7412 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7413 VMMRZCallRing3RemoveNotification(pVCpu);
7414 VMMRZCallRing3Enable(pVCpu);
7415
7416 return rc;
7417}
7418
7419
7420/**
7421 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7422 * longjump to ring-3 and possibly get preempted.
7423 *
7424 * @returns VBox status code.
7425 * @param pVCpu The cross context virtual CPU structure.
7426 * @param enmOperation The operation causing the ring-3 longjump.
7427 * @param pvUser Opaque pointer to the guest-CPU context. The data
7428 * may be out-of-sync. Make sure to update the required
7429 * fields before using them.
7430 */
7431static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7432{
7433 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7434 {
7435 /*
7436 * !!! IMPORTANT !!!
7437 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7438 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7439 */
7440 VMMRZCallRing3RemoveNotification(pVCpu);
7441 VMMRZCallRing3Disable(pVCpu);
7442 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7443 RTThreadPreemptDisable(&PreemptState);
7444
7445 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7446 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7447
7448#if HC_ARCH_BITS == 64
7449 /* Restore host-state bits that VT-x only restores partially. */
7450 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7451 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7452 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7453 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7454#endif
7455 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7456 if (pVCpu->hm.s.vmx.fLazyMsrs)
7457 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7458
7459 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7460 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7461 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7462 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7463 {
7464 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7465 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7466 }
7467
7468 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7469 VMMR0ThreadCtxHookDisable(pVCpu);
7470 HMR0LeaveCpu(pVCpu);
7471 RTThreadPreemptRestore(&PreemptState);
7472 return VINF_SUCCESS;
7473 }
7474
7475 Assert(pVCpu);
7476 Assert(pvUser);
7477 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7478 HMVMX_ASSERT_PREEMPT_SAFE();
7479
7480 VMMRZCallRing3Disable(pVCpu);
7481 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7482
7483 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7484 enmOperation));
7485
7486 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7487 AssertRCReturn(rc, rc);
7488
7489 VMMRZCallRing3Enable(pVCpu);
7490 return VINF_SUCCESS;
7491}
7492
7493
7494/**
7495 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7496 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7497 *
7498 * @param pVCpu The cross context virtual CPU structure.
7499 */
7500DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7501{
7502 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7503 {
7504 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7505 {
7506 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7507 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7508 AssertRC(rc);
7509 Log4(("Setup interrupt-window exiting\n"));
7510 }
7511 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7512}
7513
7514
7515/**
7516 * Clears the interrupt-window exiting control in the VMCS.
7517 *
7518 * @param pVCpu The cross context virtual CPU structure.
7519 */
7520DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7521{
7522 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7523 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7524 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7525 AssertRC(rc);
7526 Log4(("Cleared interrupt-window exiting\n"));
7527}
7528
7529
7530/**
7531 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7532 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7533 *
7534 * @param pVCpu The cross context virtual CPU structure.
7535 */
7536DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7537{
7538 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7539 {
7540 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7541 {
7542 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7543 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7544 AssertRC(rc);
7545 Log4(("Setup NMI-window exiting\n"));
7546 }
7547 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7548}
7549
7550
7551/**
7552 * Clears the NMI-window exiting control in the VMCS.
7553 *
7554 * @param pVCpu The cross context virtual CPU structure.
7555 */
7556DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7557{
7558 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7559 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7560 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7561 AssertRC(rc);
7562 Log4(("Cleared NMI-window exiting\n"));
7563}
7564
7565
7566/**
7567 * Evaluates the event to be delivered to the guest and sets it as the pending
7568 * event.
7569 *
7570 * @returns The VT-x guest-interruptibility state.
7571 * @param pVCpu The cross context virtual CPU structure.
7572 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7573 * out-of-sync. Make sure to update the required fields
7574 * before using them.
7575 */
7576static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7577{
7578 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7579 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7580 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7581 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7582 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7583
7584 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7585 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7586 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7587 Assert(!TRPMHasTrap(pVCpu));
7588
7589 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7590 APICUpdatePendingInterrupts(pVCpu);
7591
7592 /*
7593 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7594 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7595 */
7596 /** @todo SMI. SMIs take priority over NMIs. */
7597 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7598 {
7599 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7600 if ( !pVCpu->hm.s.Event.fPending
7601 && !fBlockNmi
7602 && !fBlockSti
7603 && !fBlockMovSS)
7604 {
7605 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7606 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7607 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7608
7609 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7610 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7611 }
7612 else
7613 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7614 }
7615 /*
7616 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7617 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7618 */
7619 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7620 && !pVCpu->hm.s.fSingleInstruction)
7621 {
7622 Assert(!DBGFIsStepping(pVCpu));
7623 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7624 AssertRC(rc);
7625 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7626 if ( !pVCpu->hm.s.Event.fPending
7627 && !fBlockInt
7628 && !fBlockSti
7629 && !fBlockMovSS)
7630 {
7631 uint8_t u8Interrupt;
7632 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7633 if (RT_SUCCESS(rc))
7634 {
7635 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7636 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7637 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7638
7639 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7640 }
7641 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7642 {
7643 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7644 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7645 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7646
7647 /*
7648 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7649 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7650 * need to re-set this force-flag here.
7651 */
7652 }
7653 else
7654 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7655 }
7656 else
7657 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7658 }
7659
7660 return uIntrState;
7661}
7662
7663
7664/**
7665 * Sets a pending-debug exception to be delivered to the guest if the guest is
7666 * single-stepping in the VMCS.
7667 *
7668 * @param pVCpu The cross context virtual CPU structure.
7669 */
7670DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7671{
7672 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7673 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7674 AssertRC(rc);
7675}
7676
7677
7678/**
7679 * Injects any pending events into the guest if the guest is in a state to
7680 * receive them.
7681 *
7682 * @returns Strict VBox status code (i.e. informational status codes too).
7683 * @param pVCpu The cross context virtual CPU structure.
7684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7685 * out-of-sync. Make sure to update the required fields
7686 * before using them.
7687 * @param uIntrState The VT-x guest-interruptibility state.
7688 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7689 * return VINF_EM_DBG_STEPPED if the event was
7690 * dispatched directly.
7691 */
7692static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7693{
7694 HMVMX_ASSERT_PREEMPT_SAFE();
7695 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7696
7697 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7698 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7699
7700 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7701 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7702 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7703 Assert(!TRPMHasTrap(pVCpu));
7704
7705 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7706 if (pVCpu->hm.s.Event.fPending)
7707 {
7708 /*
7709 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7710 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7711 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7712 *
7713 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7714 */
7715 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7716#ifdef VBOX_STRICT
7717 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7718 {
7719 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7720 Assert(!fBlockInt);
7721 Assert(!fBlockSti);
7722 Assert(!fBlockMovSS);
7723 }
7724 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7725 {
7726 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7727 Assert(!fBlockSti);
7728 Assert(!fBlockMovSS);
7729 Assert(!fBlockNmi);
7730 }
7731#endif
7732 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7733 (uint8_t)uIntType));
7734 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7735 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7736 fStepping, &uIntrState);
7737 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7738
7739 /* Update the interruptibility-state as it could have been changed by
7740 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7741 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7742 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7743
7744 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7745 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7746 else
7747 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7748 }
7749
7750 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7751 if ( fBlockSti
7752 || fBlockMovSS)
7753 {
7754 if (!pVCpu->hm.s.fSingleInstruction)
7755 {
7756 /*
7757 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7758 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7759 * See Intel spec. 27.3.4 "Saving Non-Register State".
7760 */
7761 Assert(!DBGFIsStepping(pVCpu));
7762 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7763 AssertRCReturn(rc2, rc2);
7764 if (pMixedCtx->eflags.Bits.u1TF)
7765 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7766 }
7767 else if (pMixedCtx->eflags.Bits.u1TF)
7768 {
7769 /*
7770 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7771 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7772 */
7773 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7774 uIntrState = 0;
7775 }
7776 }
7777
7778 /*
7779 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7780 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7781 */
7782 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7783 AssertRC(rc2);
7784
7785 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7786 NOREF(fBlockMovSS); NOREF(fBlockSti);
7787 return rcStrict;
7788}
7789
7790
7791/**
7792 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7793 *
7794 * @param pVCpu The cross context virtual CPU structure.
7795 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7796 * out-of-sync. Make sure to update the required fields
7797 * before using them.
7798 */
7799DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7800{
7801 NOREF(pMixedCtx);
7802 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7803 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7804}
7805
7806
7807/**
7808 * Injects a double-fault (\#DF) exception into the VM.
7809 *
7810 * @returns Strict VBox status code (i.e. informational status codes too).
7811 * @param pVCpu The cross context virtual CPU structure.
7812 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7813 * out-of-sync. Make sure to update the required fields
7814 * before using them.
7815 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7816 * and should return VINF_EM_DBG_STEPPED if the event
7817 * is injected directly (register modified by us, not
7818 * by hardware on VM-entry).
7819 * @param puIntrState Pointer to the current guest interruptibility-state.
7820 * This interruptibility-state will be updated if
7821 * necessary. This cannot not be NULL.
7822 */
7823DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7824{
7825 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7826 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7827 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7828 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7829 fStepping, puIntrState);
7830}
7831
7832
7833/**
7834 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7835 *
7836 * @param pVCpu The cross context virtual CPU structure.
7837 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7838 * out-of-sync. Make sure to update the required fields
7839 * before using them.
7840 */
7841DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7842{
7843 NOREF(pMixedCtx);
7844 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7845 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7846 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7847}
7848
7849
7850/**
7851 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7852 *
7853 * @param pVCpu The cross context virtual CPU structure.
7854 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7855 * out-of-sync. Make sure to update the required fields
7856 * before using them.
7857 * @param cbInstr The value of RIP that is to be pushed on the guest
7858 * stack.
7859 */
7860DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7861{
7862 NOREF(pMixedCtx);
7863 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7864 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7865 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7866}
7867
7868
7869/**
7870 * Injects a general-protection (\#GP) fault into the VM.
7871 *
7872 * @returns Strict VBox status code (i.e. informational status codes too).
7873 * @param pVCpu The cross context virtual CPU structure.
7874 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7875 * out-of-sync. Make sure to update the required fields
7876 * before using them.
7877 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7878 * mode, i.e. in real-mode it's not valid).
7879 * @param u32ErrorCode The error code associated with the \#GP.
7880 * @param fStepping Whether we're running in
7881 * hmR0VmxRunGuestCodeStep() and should return
7882 * VINF_EM_DBG_STEPPED if the event is injected
7883 * directly (register modified by us, not by
7884 * hardware on VM-entry).
7885 * @param puIntrState Pointer to the current guest interruptibility-state.
7886 * This interruptibility-state will be updated if
7887 * necessary. This cannot not be NULL.
7888 */
7889DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7890 bool fStepping, uint32_t *puIntrState)
7891{
7892 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7893 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7894 if (fErrorCodeValid)
7895 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7896 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7897 fStepping, puIntrState);
7898}
7899
7900
7901#if 0 /* unused */
7902/**
7903 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7904 * VM.
7905 *
7906 * @param pVCpu The cross context virtual CPU structure.
7907 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7908 * out-of-sync. Make sure to update the required fields
7909 * before using them.
7910 * @param u32ErrorCode The error code associated with the \#GP.
7911 */
7912DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7913{
7914 NOREF(pMixedCtx);
7915 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7916 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7917 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7918 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7919}
7920#endif /* unused */
7921
7922
7923/**
7924 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7925 *
7926 * @param pVCpu The cross context virtual CPU structure.
7927 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7928 * out-of-sync. Make sure to update the required fields
7929 * before using them.
7930 * @param uVector The software interrupt vector number.
7931 * @param cbInstr The value of RIP that is to be pushed on the guest
7932 * stack.
7933 */
7934DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7935{
7936 NOREF(pMixedCtx);
7937 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7938 if ( uVector == X86_XCPT_BP
7939 || uVector == X86_XCPT_OF)
7940 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7941 else
7942 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7943 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7944}
7945
7946
7947/**
7948 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7949 * stack.
7950 *
7951 * @returns Strict VBox status code (i.e. informational status codes too).
7952 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7953 * @param pVM The cross context VM structure.
7954 * @param pMixedCtx Pointer to the guest-CPU context.
7955 * @param uValue The value to push to the guest stack.
7956 */
7957DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7958{
7959 /*
7960 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7961 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7962 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7963 */
7964 if (pMixedCtx->sp == 1)
7965 return VINF_EM_RESET;
7966 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7967 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7968 AssertRC(rc);
7969 return rc;
7970}
7971
7972
7973/**
7974 * Injects an event into the guest upon VM-entry by updating the relevant fields
7975 * in the VM-entry area in the VMCS.
7976 *
7977 * @returns Strict VBox status code (i.e. informational status codes too).
7978 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7979 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7980 *
7981 * @param pVCpu The cross context virtual CPU structure.
7982 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7983 * be out-of-sync. Make sure to update the required
7984 * fields before using them.
7985 * @param u64IntInfo The VM-entry interruption-information field.
7986 * @param cbInstr The VM-entry instruction length in bytes (for
7987 * software interrupts, exceptions and privileged
7988 * software exceptions).
7989 * @param u32ErrCode The VM-entry exception error code.
7990 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7991 * @param puIntrState Pointer to the current guest interruptibility-state.
7992 * This interruptibility-state will be updated if
7993 * necessary. This cannot not be NULL.
7994 * @param fStepping Whether we're running in
7995 * hmR0VmxRunGuestCodeStep() and should return
7996 * VINF_EM_DBG_STEPPED if the event is injected
7997 * directly (register modified by us, not by
7998 * hardware on VM-entry).
7999 *
8000 * @remarks Requires CR0!
8001 * @remarks No-long-jump zone!!!
8002 */
8003static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8004 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8005 uint32_t *puIntrState)
8006{
8007 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8008 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8009 Assert(puIntrState);
8010 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8011
8012 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8013 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8014
8015#ifdef VBOX_STRICT
8016 /* Validate the error-code-valid bit for hardware exceptions. */
8017 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8018 {
8019 switch (uVector)
8020 {
8021 case X86_XCPT_PF:
8022 case X86_XCPT_DF:
8023 case X86_XCPT_TS:
8024 case X86_XCPT_NP:
8025 case X86_XCPT_SS:
8026 case X86_XCPT_GP:
8027 case X86_XCPT_AC:
8028 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8029 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8030 /* fallthru */
8031 default:
8032 break;
8033 }
8034 }
8035#endif
8036
8037 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8038 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8039 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8040
8041 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8042
8043 /* We require CR0 to check if the guest is in real-mode. */
8044 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8045 AssertRCReturn(rc, rc);
8046
8047 /*
8048 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8049 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8050 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8051 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8052 */
8053 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8054 {
8055 PVM pVM = pVCpu->CTX_SUFF(pVM);
8056 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8057 {
8058 Assert(PDMVmmDevHeapIsEnabled(pVM));
8059 Assert(pVM->hm.s.vmx.pRealModeTSS);
8060
8061 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8062 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8063 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8064 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8065 AssertRCReturn(rc, rc);
8066 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8067
8068 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8069 size_t const cbIdtEntry = sizeof(X86IDTR16);
8070 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8071 {
8072 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8073 if (uVector == X86_XCPT_DF)
8074 return VINF_EM_RESET;
8075
8076 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8077 if (uVector == X86_XCPT_GP)
8078 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8079
8080 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8081 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8082 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8083 fStepping, puIntrState);
8084 }
8085
8086 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8087 uint16_t uGuestIp = pMixedCtx->ip;
8088 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8089 {
8090 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8091 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8092 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8093 }
8094 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8095 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8096
8097 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8098 X86IDTR16 IdtEntry;
8099 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8100 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8101 AssertRCReturn(rc, rc);
8102
8103 /* Construct the stack frame for the interrupt/exception handler. */
8104 VBOXSTRICTRC rcStrict;
8105 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8106 if (rcStrict == VINF_SUCCESS)
8107 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8108 if (rcStrict == VINF_SUCCESS)
8109 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8110
8111 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8112 if (rcStrict == VINF_SUCCESS)
8113 {
8114 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8115 pMixedCtx->rip = IdtEntry.offSel;
8116 pMixedCtx->cs.Sel = IdtEntry.uSel;
8117 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8118 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8119 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8120 && uVector == X86_XCPT_PF)
8121 pMixedCtx->cr2 = GCPtrFaultAddress;
8122
8123 /* If any other guest-state bits are changed here, make sure to update
8124 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8125 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8126 | HM_CHANGED_GUEST_RIP
8127 | HM_CHANGED_GUEST_RFLAGS
8128 | HM_CHANGED_GUEST_RSP);
8129
8130 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8131 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8132 {
8133 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8134 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8135 Log4(("Clearing inhibition due to STI.\n"));
8136 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8137 }
8138 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8139 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8140
8141 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8142 it, if we are returning to ring-3 before executing guest code. */
8143 pVCpu->hm.s.Event.fPending = false;
8144
8145 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8146 if (fStepping)
8147 rcStrict = VINF_EM_DBG_STEPPED;
8148 }
8149 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8150 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8151 return rcStrict;
8152 }
8153
8154 /*
8155 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8156 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8157 */
8158 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8159 }
8160
8161 /* Validate. */
8162 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8163 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8164 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8165
8166 /* Inject. */
8167 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8168 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8169 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8170 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8171
8172 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8173 && uVector == X86_XCPT_PF)
8174 pMixedCtx->cr2 = GCPtrFaultAddress;
8175
8176 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8177 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8178
8179 AssertRCReturn(rc, rc);
8180 return VINF_SUCCESS;
8181}
8182
8183
8184/**
8185 * Clears the interrupt-window exiting control in the VMCS and if necessary
8186 * clears the current event in the VMCS as well.
8187 *
8188 * @returns VBox status code.
8189 * @param pVCpu The cross context virtual CPU structure.
8190 *
8191 * @remarks Use this function only to clear events that have not yet been
8192 * delivered to the guest but are injected in the VMCS!
8193 * @remarks No-long-jump zone!!!
8194 */
8195static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8196{
8197 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8198
8199 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8200 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8201
8202 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8203 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8204}
8205
8206
8207/**
8208 * Enters the VT-x session.
8209 *
8210 * @returns VBox status code.
8211 * @param pVM The cross context VM structure.
8212 * @param pVCpu The cross context virtual CPU structure.
8213 * @param pCpu Pointer to the CPU info struct.
8214 */
8215VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8216{
8217 AssertPtr(pVM);
8218 AssertPtr(pVCpu);
8219 Assert(pVM->hm.s.vmx.fSupported);
8220 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8221 NOREF(pCpu); NOREF(pVM);
8222
8223 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8224 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8225
8226#ifdef VBOX_STRICT
8227 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8228 RTCCUINTREG uHostCR4 = ASMGetCR4();
8229 if (!(uHostCR4 & X86_CR4_VMXE))
8230 {
8231 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8232 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8233 }
8234#endif
8235
8236 /*
8237 * Load the VCPU's VMCS as the current (and active) one.
8238 */
8239 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8240 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8241 if (RT_FAILURE(rc))
8242 return rc;
8243
8244 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8245 pVCpu->hm.s.fLeaveDone = false;
8246 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8247
8248 return VINF_SUCCESS;
8249}
8250
8251
8252/**
8253 * The thread-context callback (only on platforms which support it).
8254 *
8255 * @param enmEvent The thread-context event.
8256 * @param pVCpu The cross context virtual CPU structure.
8257 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8258 * @thread EMT(pVCpu)
8259 */
8260VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8261{
8262 NOREF(fGlobalInit);
8263
8264 switch (enmEvent)
8265 {
8266 case RTTHREADCTXEVENT_OUT:
8267 {
8268 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8269 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8270 VMCPU_ASSERT_EMT(pVCpu);
8271
8272 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8273
8274 /* No longjmps (logger flushes, locks) in this fragile context. */
8275 VMMRZCallRing3Disable(pVCpu);
8276 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8277
8278 /*
8279 * Restore host-state (FPU, debug etc.)
8280 */
8281 if (!pVCpu->hm.s.fLeaveDone)
8282 {
8283 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8284 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8285 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8286 pVCpu->hm.s.fLeaveDone = true;
8287 }
8288
8289 /* Leave HM context, takes care of local init (term). */
8290 int rc = HMR0LeaveCpu(pVCpu);
8291 AssertRC(rc); NOREF(rc);
8292
8293 /* Restore longjmp state. */
8294 VMMRZCallRing3Enable(pVCpu);
8295 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8296 break;
8297 }
8298
8299 case RTTHREADCTXEVENT_IN:
8300 {
8301 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8302 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8303 VMCPU_ASSERT_EMT(pVCpu);
8304
8305 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8306 VMMRZCallRing3Disable(pVCpu);
8307 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8308
8309 /* Initialize the bare minimum state required for HM. This takes care of
8310 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8311 int rc = HMR0EnterCpu(pVCpu);
8312 AssertRC(rc);
8313 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8314
8315 /* Load the active VMCS as the current one. */
8316 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8317 {
8318 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8319 AssertRC(rc); NOREF(rc);
8320 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8321 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8322 }
8323 pVCpu->hm.s.fLeaveDone = false;
8324
8325 /* Restore longjmp state. */
8326 VMMRZCallRing3Enable(pVCpu);
8327 break;
8328 }
8329
8330 default:
8331 break;
8332 }
8333}
8334
8335
8336/**
8337 * Saves the host state in the VMCS host-state.
8338 * Sets up the VM-exit MSR-load area.
8339 *
8340 * The CPU state will be loaded from these fields on every successful VM-exit.
8341 *
8342 * @returns VBox status code.
8343 * @param pVM The cross context VM structure.
8344 * @param pVCpu The cross context virtual CPU structure.
8345 *
8346 * @remarks No-long-jump zone!!!
8347 */
8348static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8349{
8350 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8351
8352 int rc = VINF_SUCCESS;
8353 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8354 {
8355 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8356 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8357
8358 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8359 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8360
8361 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8362 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8363
8364 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8365 }
8366 return rc;
8367}
8368
8369
8370/**
8371 * Saves the host state in the VMCS host-state.
8372 *
8373 * @returns VBox status code.
8374 * @param pVM The cross context VM structure.
8375 * @param pVCpu The cross context virtual CPU structure.
8376 *
8377 * @remarks No-long-jump zone!!!
8378 */
8379VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8380{
8381 AssertPtr(pVM);
8382 AssertPtr(pVCpu);
8383
8384 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8385
8386 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8387 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8388 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8389 return hmR0VmxSaveHostState(pVM, pVCpu);
8390}
8391
8392
8393/**
8394 * Loads the guest state into the VMCS guest-state area.
8395 *
8396 * The will typically be done before VM-entry when the guest-CPU state and the
8397 * VMCS state may potentially be out of sync.
8398 *
8399 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8400 * VM-entry controls.
8401 * Sets up the appropriate VMX non-root function to execute guest code based on
8402 * the guest CPU mode.
8403 *
8404 * @returns VBox strict status code.
8405 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8406 * without unrestricted guest access and the VMMDev is not presently
8407 * mapped (e.g. EFI32).
8408 *
8409 * @param pVM The cross context VM structure.
8410 * @param pVCpu The cross context virtual CPU structure.
8411 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8412 * out-of-sync. Make sure to update the required fields
8413 * before using them.
8414 */
8415static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8416{
8417 AssertPtr(pVM);
8418 AssertPtr(pVCpu);
8419 AssertPtr(pMixedCtx);
8420 HMVMX_ASSERT_PREEMPT_SAFE();
8421
8422 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8423
8424 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8425
8426 /* Determine real-on-v86 mode. */
8427 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8428 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8429 && CPUMIsGuestInRealModeEx(pMixedCtx))
8430 {
8431 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8432 }
8433
8434 /*
8435 * Load the guest-state into the VMCS.
8436 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8437 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8438 */
8439 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8440 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8441
8442 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8443 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8444 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8445
8446 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8447 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8448 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8449
8450 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8451 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8452
8453 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8454 if (rcStrict == VINF_SUCCESS)
8455 { /* likely */ }
8456 else
8457 {
8458 VMMRZCallRing3Enable(pVCpu);
8459 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8460 return rcStrict;
8461 }
8462
8463 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8464 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8465 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8466
8467 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8468 determine we don't have to swap EFER after all. */
8469 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8470 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8471
8472 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8473 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8474
8475 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8476 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8477
8478 /*
8479 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8480 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8481 */
8482 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8483 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8484
8485 /* Clear any unused and reserved bits. */
8486 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8487
8488 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8489 return rc;
8490}
8491
8492
8493/**
8494 * Loads the state shared between the host and guest into the VMCS.
8495 *
8496 * @param pVM The cross context VM structure.
8497 * @param pVCpu The cross context virtual CPU structure.
8498 * @param pCtx Pointer to the guest-CPU context.
8499 *
8500 * @remarks No-long-jump zone!!!
8501 */
8502static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8503{
8504 NOREF(pVM);
8505
8506 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8507 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8508
8509 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8510 {
8511 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8512 AssertRC(rc);
8513 }
8514
8515 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8516 {
8517 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8518 AssertRC(rc);
8519
8520 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8521 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8522 {
8523 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8524 AssertRC(rc);
8525 }
8526 }
8527
8528 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8529 {
8530 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8531 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8532 }
8533
8534 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8535 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8536 {
8537 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8538 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8539 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8540 AssertRC(rc);
8541 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8542 }
8543
8544 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8545 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8546}
8547
8548
8549/**
8550 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8551 *
8552 * @returns Strict VBox status code (i.e. informational status codes too).
8553 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8554 * without unrestricted guest access and the VMMDev is not presently
8555 * mapped (e.g. EFI32).
8556 *
8557 * @param pVM The cross context VM structure.
8558 * @param pVCpu The cross context virtual CPU structure.
8559 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8560 * out-of-sync. Make sure to update the required fields
8561 * before using them.
8562 */
8563static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8564{
8565 HMVMX_ASSERT_PREEMPT_SAFE();
8566
8567 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8568#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8569 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8570#endif
8571
8572 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8573 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8574 {
8575 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8576 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8577 { /* likely */}
8578 else
8579 {
8580 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8581 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8582 }
8583 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8584 }
8585 else if (HMCPU_CF_VALUE(pVCpu))
8586 {
8587 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8588 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8589 { /* likely */}
8590 else
8591 {
8592 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8593 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8594 return rcStrict;
8595 }
8596 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8597 }
8598
8599 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8600 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8601 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8602 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8603 return rcStrict;
8604}
8605
8606
8607/**
8608 * Does the preparations before executing guest code in VT-x.
8609 *
8610 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8611 * recompiler/IEM. We must be cautious what we do here regarding committing
8612 * guest-state information into the VMCS assuming we assuredly execute the
8613 * guest in VT-x mode.
8614 *
8615 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8616 * the common-state (TRPM/forceflags), we must undo those changes so that the
8617 * recompiler/IEM can (and should) use them when it resumes guest execution.
8618 * Otherwise such operations must be done when we can no longer exit to ring-3.
8619 *
8620 * @returns Strict VBox status code (i.e. informational status codes too).
8621 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8622 * have been disabled.
8623 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8624 * double-fault into the guest.
8625 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8626 * dispatched directly.
8627 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8628 *
8629 * @param pVM The cross context VM structure.
8630 * @param pVCpu The cross context virtual CPU structure.
8631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8632 * out-of-sync. Make sure to update the required fields
8633 * before using them.
8634 * @param pVmxTransient Pointer to the VMX transient structure.
8635 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8636 * us ignore some of the reasons for returning to
8637 * ring-3, and return VINF_EM_DBG_STEPPED if event
8638 * dispatching took place.
8639 */
8640static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8641{
8642 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8643
8644#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8645 PGMRZDynMapFlushAutoSet(pVCpu);
8646#endif
8647
8648 /* Check force flag actions that might require us to go back to ring-3. */
8649 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8650 if (rcStrict == VINF_SUCCESS)
8651 { /* FFs doesn't get set all the time. */ }
8652 else
8653 return rcStrict;
8654
8655 if (TRPMHasTrap(pVCpu))
8656 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8657 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8658
8659 /*
8660 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8661 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8662 */
8663 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8664 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8665 { /* likely */ }
8666 else
8667 {
8668 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8669 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8670 return rcStrict;
8671 }
8672
8673 /*
8674 * Load the guest state bits, we can handle longjmps/getting preempted here.
8675 *
8676 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8677 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8678 * Hence, this needs to be done -after- injection of events.
8679 */
8680 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8681 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8682 { /* likely */ }
8683 else
8684 return rcStrict;
8685
8686 /*
8687 * No longjmps to ring-3 from this point on!!!
8688 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8689 * This also disables flushing of the R0-logger instance (if any).
8690 */
8691 VMMRZCallRing3Disable(pVCpu);
8692
8693 /*
8694 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8695 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8696 *
8697 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8698 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8699 *
8700 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8701 * executing guest code.
8702 */
8703 pVmxTransient->fEFlags = ASMIntDisableFlags();
8704
8705 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8706 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8707 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8708 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8709 {
8710 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8711 {
8712 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8713 pVCpu->hm.s.Event.fPending = false;
8714
8715 return VINF_SUCCESS;
8716 }
8717
8718 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8719 rcStrict = VINF_EM_RAW_INTERRUPT;
8720 }
8721 else
8722 {
8723 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8724 rcStrict = VINF_EM_RAW_TO_R3;
8725 }
8726
8727 ASMSetFlags(pVmxTransient->fEFlags);
8728 VMMRZCallRing3Enable(pVCpu);
8729
8730 return rcStrict;
8731}
8732
8733
8734/**
8735 * Prepares to run guest code in VT-x and we've committed to doing so. This
8736 * means there is no backing out to ring-3 or anywhere else at this
8737 * point.
8738 *
8739 * @param pVM The cross context VM structure.
8740 * @param pVCpu The cross context virtual CPU structure.
8741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8742 * out-of-sync. Make sure to update the required fields
8743 * before using them.
8744 * @param pVmxTransient Pointer to the VMX transient structure.
8745 *
8746 * @remarks Called with preemption disabled.
8747 * @remarks No-long-jump zone!!!
8748 */
8749static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8750{
8751 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8752 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8753 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8754
8755 /*
8756 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8757 */
8758 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8759 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8760
8761#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8762 if (!CPUMIsGuestFPUStateActive(pVCpu))
8763 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8764 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8765 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8766#endif
8767
8768 if ( pVCpu->hm.s.fPreloadGuestFpu
8769 && !CPUMIsGuestFPUStateActive(pVCpu))
8770 {
8771 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8772 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8773 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8774 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8775 }
8776
8777 /*
8778 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8779 */
8780 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8781 && pVCpu->hm.s.vmx.cMsrs > 0)
8782 {
8783 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8784 }
8785
8786 /*
8787 * Load the host state bits as we may've been preempted (only happens when
8788 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8789 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8790 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8791 * See @bugref{8432}.
8792 */
8793 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8794 {
8795 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8796 AssertRC(rc);
8797 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8798 }
8799 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8800
8801 /*
8802 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8803 */
8804 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8805 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8806 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8807
8808 /* Store status of the shared guest-host state at the time of VM-entry. */
8809#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8810 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8811 {
8812 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8813 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8814 }
8815 else
8816#endif
8817 {
8818 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8819 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8820 }
8821 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8822
8823 /*
8824 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8825 */
8826 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8827 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8828
8829 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8830 RTCPUID idCurrentCpu = pCpu->idCpu;
8831 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8832 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8833 {
8834 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8835 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8836 }
8837
8838 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8839 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8840 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8841 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8842
8843 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8844
8845 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8846 to start executing. */
8847
8848 /*
8849 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8850 */
8851 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8852 {
8853 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8854 {
8855 bool fMsrUpdated;
8856 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8857 AssertRC(rc2);
8858 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8859
8860 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8861 &fMsrUpdated);
8862 AssertRC(rc2);
8863 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8864
8865 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8866 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8867 }
8868 else
8869 {
8870 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8871 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8872 }
8873 }
8874
8875#ifdef VBOX_STRICT
8876 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8877 hmR0VmxCheckHostEferMsr(pVCpu);
8878 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8879#endif
8880#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8881 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8882 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8883 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8884#endif
8885}
8886
8887
8888/**
8889 * Performs some essential restoration of state after running guest code in
8890 * VT-x.
8891 *
8892 * @param pVM The cross context VM structure.
8893 * @param pVCpu The cross context virtual CPU structure.
8894 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8895 * out-of-sync. Make sure to update the required fields
8896 * before using them.
8897 * @param pVmxTransient Pointer to the VMX transient structure.
8898 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8899 *
8900 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8901 *
8902 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8903 * unconditionally when it is safe to do so.
8904 */
8905static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8906{
8907 NOREF(pVM);
8908
8909 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8910
8911 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8912 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8913 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8914 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8915 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8916 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8917
8918 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8919 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8920
8921 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8922 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8923 Assert(!ASMIntAreEnabled());
8924 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8925
8926#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8927 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8928 {
8929 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8930 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8931 }
8932#endif
8933
8934#if HC_ARCH_BITS == 64
8935 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8936#endif
8937#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8938 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8939 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8940 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8941#else
8942 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8943#endif
8944#ifdef VBOX_STRICT
8945 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8946#endif
8947 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8948 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8949
8950 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8951 uint32_t uExitReason;
8952 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8953 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8954 AssertRC(rc);
8955 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8956 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8957
8958 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8959 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8960 {
8961 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8962 pVmxTransient->fVMEntryFailed));
8963 return;
8964 }
8965
8966 /*
8967 * Update the VM-exit history array here even if the VM-entry failed due to:
8968 * - Invalid guest state.
8969 * - MSR loading.
8970 * - Machine-check event.
8971 *
8972 * In any of the above cases we will still have a "valid" VM-exit reason
8973 * despite @a fVMEntryFailed being false.
8974 *
8975 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8976 */
8977 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8978
8979 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8980 {
8981 /** @todo We can optimize this by only syncing with our force-flags when
8982 * really needed and keeping the VMCS state as it is for most
8983 * VM-exits. */
8984 /* Update the guest interruptibility-state from the VMCS. */
8985 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8986
8987#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8988 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8989 AssertRC(rc);
8990#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8991 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8992 AssertRC(rc);
8993#endif
8994
8995 /*
8996 * Sync the TPR shadow with our APIC state.
8997 */
8998 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8999 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9000 {
9001 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9002 AssertRC(rc);
9003 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9004 }
9005 }
9006}
9007
9008
9009/**
9010 * Runs the guest code using VT-x the normal way.
9011 *
9012 * @returns VBox status code.
9013 * @param pVM The cross context VM structure.
9014 * @param pVCpu The cross context virtual CPU structure.
9015 * @param pCtx Pointer to the guest-CPU context.
9016 *
9017 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9018 */
9019static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9020{
9021 VMXTRANSIENT VmxTransient;
9022 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9023 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9024 uint32_t cLoops = 0;
9025
9026 for (;; cLoops++)
9027 {
9028 Assert(!HMR0SuspendPending());
9029 HMVMX_ASSERT_CPU_SAFE();
9030
9031 /* Preparatory work for running guest code, this may force us to return
9032 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9033 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9034 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9035 if (rcStrict != VINF_SUCCESS)
9036 break;
9037
9038 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9039 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9040 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9041
9042 /* Restore any residual host-state and save any bits shared between host
9043 and guest into the guest-CPU state. Re-enables interrupts! */
9044 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9045
9046 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9047 if (RT_SUCCESS(rcRun))
9048 { /* very likely */ }
9049 else
9050 {
9051 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9052 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9053 return rcRun;
9054 }
9055
9056 /* Profile the VM-exit. */
9057 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9059 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9060 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9061 HMVMX_START_EXIT_DISPATCH_PROF();
9062
9063 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9064
9065 /* Handle the VM-exit. */
9066#ifdef HMVMX_USE_FUNCTION_TABLE
9067 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9068#else
9069 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9070#endif
9071 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9072 if (rcStrict == VINF_SUCCESS)
9073 {
9074 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9075 continue; /* likely */
9076 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9077 rcStrict = VINF_EM_RAW_INTERRUPT;
9078 }
9079 break;
9080 }
9081
9082 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9083 return rcStrict;
9084}
9085
9086
9087
9088/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9089 * probes.
9090 *
9091 * The following few functions and associated structure contains the bloat
9092 * necessary for providing detailed debug events and dtrace probes as well as
9093 * reliable host side single stepping. This works on the principle of
9094 * "subclassing" the normal execution loop and workers. We replace the loop
9095 * method completely and override selected helpers to add necessary adjustments
9096 * to their core operation.
9097 *
9098 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9099 * any performance for debug and analysis features.
9100 *
9101 * @{
9102 */
9103
9104/**
9105 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9106 * the debug run loop.
9107 */
9108typedef struct VMXRUNDBGSTATE
9109{
9110 /** The RIP we started executing at. This is for detecting that we stepped. */
9111 uint64_t uRipStart;
9112 /** The CS we started executing with. */
9113 uint16_t uCsStart;
9114
9115 /** Whether we've actually modified the 1st execution control field. */
9116 bool fModifiedProcCtls : 1;
9117 /** Whether we've actually modified the 2nd execution control field. */
9118 bool fModifiedProcCtls2 : 1;
9119 /** Whether we've actually modified the exception bitmap. */
9120 bool fModifiedXcptBitmap : 1;
9121
9122 /** We desire the modified the CR0 mask to be cleared. */
9123 bool fClearCr0Mask : 1;
9124 /** We desire the modified the CR4 mask to be cleared. */
9125 bool fClearCr4Mask : 1;
9126 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9127 uint32_t fCpe1Extra;
9128 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9129 uint32_t fCpe1Unwanted;
9130 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9131 uint32_t fCpe2Extra;
9132 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9133 uint32_t bmXcptExtra;
9134 /** The sequence number of the Dtrace provider settings the state was
9135 * configured against. */
9136 uint32_t uDtraceSettingsSeqNo;
9137 /** VM-exits to check (one bit per VM-exit). */
9138 uint32_t bmExitsToCheck[3];
9139
9140 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9141 uint32_t fProcCtlsInitial;
9142 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9143 uint32_t fProcCtls2Initial;
9144 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9145 uint32_t bmXcptInitial;
9146} VMXRUNDBGSTATE;
9147AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9148typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9149
9150
9151/**
9152 * Initializes the VMXRUNDBGSTATE structure.
9153 *
9154 * @param pVCpu The cross context virtual CPU structure of the
9155 * calling EMT.
9156 * @param pCtx The CPU register context to go with @a pVCpu.
9157 * @param pDbgState The structure to initialize.
9158 */
9159DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9160{
9161 pDbgState->uRipStart = pCtx->rip;
9162 pDbgState->uCsStart = pCtx->cs.Sel;
9163
9164 pDbgState->fModifiedProcCtls = false;
9165 pDbgState->fModifiedProcCtls2 = false;
9166 pDbgState->fModifiedXcptBitmap = false;
9167 pDbgState->fClearCr0Mask = false;
9168 pDbgState->fClearCr4Mask = false;
9169 pDbgState->fCpe1Extra = 0;
9170 pDbgState->fCpe1Unwanted = 0;
9171 pDbgState->fCpe2Extra = 0;
9172 pDbgState->bmXcptExtra = 0;
9173 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9174 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9175 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9176}
9177
9178
9179/**
9180 * Updates the VMSC fields with changes requested by @a pDbgState.
9181 *
9182 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9183 * immediately before executing guest code, i.e. when interrupts are disabled.
9184 * We don't check status codes here as we cannot easily assert or return in the
9185 * latter case.
9186 *
9187 * @param pVCpu The cross context virtual CPU structure.
9188 * @param pDbgState The debug state.
9189 */
9190DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9191{
9192 /*
9193 * Ensure desired flags in VMCS control fields are set.
9194 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9195 *
9196 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9197 * there should be no stale data in pCtx at this point.
9198 */
9199 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9200 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9201 {
9202 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9203 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9204 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9205 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9206 pDbgState->fModifiedProcCtls = true;
9207 }
9208
9209 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9210 {
9211 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9212 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9213 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9214 pDbgState->fModifiedProcCtls2 = true;
9215 }
9216
9217 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9218 {
9219 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9220 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9221 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9222 pDbgState->fModifiedXcptBitmap = true;
9223 }
9224
9225 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9226 {
9227 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9228 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9229 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9230 }
9231
9232 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9233 {
9234 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9235 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9236 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9237 }
9238}
9239
9240
9241DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9242{
9243 /*
9244 * Restore VM-exit control settings as we may not reenter this function the
9245 * next time around.
9246 */
9247 /* We reload the initial value, trigger what we can of recalculations the
9248 next time around. From the looks of things, that's all that's required atm. */
9249 if (pDbgState->fModifiedProcCtls)
9250 {
9251 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9252 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9253 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9254 AssertRCReturn(rc2, rc2);
9255 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9256 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9257 }
9258
9259 /* We're currently the only ones messing with this one, so just restore the
9260 cached value and reload the field. */
9261 if ( pDbgState->fModifiedProcCtls2
9262 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9263 {
9264 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9265 AssertRCReturn(rc2, rc2);
9266 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9267 }
9268
9269 /* If we've modified the exception bitmap, we restore it and trigger
9270 reloading and partial recalculation the next time around. */
9271 if (pDbgState->fModifiedXcptBitmap)
9272 {
9273 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9274 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9275 }
9276
9277 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9278 if (pDbgState->fClearCr0Mask)
9279 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9280
9281 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9282 if (pDbgState->fClearCr4Mask)
9283 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9284
9285 return rcStrict;
9286}
9287
9288
9289/**
9290 * Configures VM-exit controls for current DBGF and DTrace settings.
9291 *
9292 * This updates @a pDbgState and the VMCS execution control fields to reflect
9293 * the necessary VM-exits demanded by DBGF and DTrace.
9294 *
9295 * @param pVM The cross context VM structure.
9296 * @param pVCpu The cross context virtual CPU structure.
9297 * @param pCtx Pointer to the guest-CPU context.
9298 * @param pDbgState The debug state.
9299 * @param pVmxTransient Pointer to the VMX transient structure. May update
9300 * fUpdateTscOffsettingAndPreemptTimer.
9301 */
9302static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9303 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9304{
9305 /*
9306 * Take down the dtrace serial number so we can spot changes.
9307 */
9308 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9309 ASMCompilerBarrier();
9310
9311 /*
9312 * We'll rebuild most of the middle block of data members (holding the
9313 * current settings) as we go along here, so start by clearing it all.
9314 */
9315 pDbgState->bmXcptExtra = 0;
9316 pDbgState->fCpe1Extra = 0;
9317 pDbgState->fCpe1Unwanted = 0;
9318 pDbgState->fCpe2Extra = 0;
9319 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9320 pDbgState->bmExitsToCheck[i] = 0;
9321
9322 /*
9323 * Software interrupts (INT XXh) - no idea how to trigger these...
9324 */
9325 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9326 || VBOXVMM_INT_SOFTWARE_ENABLED())
9327 {
9328 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9329 }
9330
9331 /*
9332 * INT3 breakpoints - triggered by #BP exceptions.
9333 */
9334 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9335 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9336
9337 /*
9338 * Exception bitmap and XCPT events+probes.
9339 */
9340 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9341 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9342 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9343
9344 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9345 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9346 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9347 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9348 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9349 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9350 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9351 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9352 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9353 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9354 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9355 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9356 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9357 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9358 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9359 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9360 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9361 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9362
9363 if (pDbgState->bmXcptExtra)
9364 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9365
9366 /*
9367 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9368 *
9369 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9370 * So, when adding/changing/removing please don't forget to update it.
9371 *
9372 * Some of the macros are picking up local variables to save horizontal space,
9373 * (being able to see it in a table is the lesser evil here).
9374 */
9375#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9376 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9377 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9378#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9379 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9380 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9381 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9382 } else do { } while (0)
9383#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9384 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9385 { \
9386 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9387 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9388 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9389 } else do { } while (0)
9390#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9391 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9392 { \
9393 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9394 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9395 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9396 } else do { } while (0)
9397#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9398 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9399 { \
9400 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9401 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9402 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9403 } else do { } while (0)
9404
9405 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9406 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9407 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9408 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9409 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9410
9411 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9412 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9413 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9415 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9417 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9419 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9421 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9423 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9425 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9431 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9433 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9435 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9437 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9439 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9441 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9443 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9444 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9445 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9446 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9447
9448 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9449 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9450 {
9451 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9452 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9453 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9454 AssertRC(rc2);
9455
9456#if 0 /** @todo fix me */
9457 pDbgState->fClearCr0Mask = true;
9458 pDbgState->fClearCr4Mask = true;
9459#endif
9460 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9461 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9462 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9463 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9464 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9465 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9466 require clearing here and in the loop if we start using it. */
9467 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9468 }
9469 else
9470 {
9471 if (pDbgState->fClearCr0Mask)
9472 {
9473 pDbgState->fClearCr0Mask = false;
9474 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9475 }
9476 if (pDbgState->fClearCr4Mask)
9477 {
9478 pDbgState->fClearCr4Mask = false;
9479 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9480 }
9481 }
9482 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9483 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9484
9485 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9486 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9487 {
9488 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9489 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9490 }
9491 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9492 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9493
9494 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9495 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9496 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9497 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9498 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9499 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9500 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9501 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9502#if 0 /** @todo too slow, fix handler. */
9503 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9504#endif
9505 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9506
9507 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9508 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9509 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9510 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9511 {
9512 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9513 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9514 }
9515 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9516 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9517 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9518 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9519
9520 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9521 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9522 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9523 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9524 {
9525 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9526 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9527 }
9528 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9529 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9530 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9531 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9532
9533 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9535 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9536 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9537 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9538 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9539 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9540 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9541 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9542 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9543 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9544 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9545 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9546 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9547 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9548 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9549 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9550 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9551 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9552 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9553 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9554 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9555
9556#undef IS_EITHER_ENABLED
9557#undef SET_ONLY_XBM_IF_EITHER_EN
9558#undef SET_CPE1_XBM_IF_EITHER_EN
9559#undef SET_CPEU_XBM_IF_EITHER_EN
9560#undef SET_CPE2_XBM_IF_EITHER_EN
9561
9562 /*
9563 * Sanitize the control stuff.
9564 */
9565 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9566 if (pDbgState->fCpe2Extra)
9567 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9568 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9569 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9570 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9571 {
9572 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9573 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9574 }
9575
9576 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9577 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9578 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9579 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9580}
9581
9582
9583/**
9584 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9585 * appropriate.
9586 *
9587 * The caller has checked the VM-exit against the
9588 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9589 * already, so we don't have to do that either.
9590 *
9591 * @returns Strict VBox status code (i.e. informational status codes too).
9592 * @param pVM The cross context VM structure.
9593 * @param pVCpu The cross context virtual CPU structure.
9594 * @param pMixedCtx Pointer to the guest-CPU context.
9595 * @param pVmxTransient Pointer to the VMX-transient structure.
9596 * @param uExitReason The VM-exit reason.
9597 *
9598 * @remarks The name of this function is displayed by dtrace, so keep it short
9599 * and to the point. No longer than 33 chars long, please.
9600 */
9601static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9602 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9603{
9604 /*
9605 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9606 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9607 *
9608 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9609 * does. Must add/change/remove both places. Same ordering, please.
9610 *
9611 * Added/removed events must also be reflected in the next section
9612 * where we dispatch dtrace events.
9613 */
9614 bool fDtrace1 = false;
9615 bool fDtrace2 = false;
9616 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9617 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9618 uint32_t uEventArg = 0;
9619#define SET_EXIT(a_EventSubName) \
9620 do { \
9621 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9622 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9623 } while (0)
9624#define SET_BOTH(a_EventSubName) \
9625 do { \
9626 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9627 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9628 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9629 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9630 } while (0)
9631 switch (uExitReason)
9632 {
9633 case VMX_EXIT_MTF:
9634 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9635
9636 case VMX_EXIT_XCPT_OR_NMI:
9637 {
9638 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9639 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9640 {
9641 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9642 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9643 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9644 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9645 {
9646 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9647 {
9648 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9649 uEventArg = pVmxTransient->uExitIntErrorCode;
9650 }
9651 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9652 switch (enmEvent1)
9653 {
9654 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9655 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9656 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9657 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9658 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9659 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9660 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9661 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9662 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9663 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9664 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9665 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9666 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9667 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9668 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9669 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9670 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9671 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9672 default: break;
9673 }
9674 }
9675 else
9676 AssertFailed();
9677 break;
9678
9679 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9680 uEventArg = idxVector;
9681 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9682 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9683 break;
9684 }
9685 break;
9686 }
9687
9688 case VMX_EXIT_TRIPLE_FAULT:
9689 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9690 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9691 break;
9692 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9693 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9694 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9695 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9696 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9697
9698 /* Instruction specific VM-exits: */
9699 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9700 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9701 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9702 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9703 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9704 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9705 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9706 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9707 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9708 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9709 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9710 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9711 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9712 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9713 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9714 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9715 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9716 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9717 case VMX_EXIT_MOV_CRX:
9718 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9719/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9720* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9721 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9722 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9723 SET_BOTH(CRX_READ);
9724 else
9725 SET_BOTH(CRX_WRITE);
9726 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9727 break;
9728 case VMX_EXIT_MOV_DRX:
9729 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9730 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9731 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9732 SET_BOTH(DRX_READ);
9733 else
9734 SET_BOTH(DRX_WRITE);
9735 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9736 break;
9737 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9738 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9739 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9740 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9741 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9742 case VMX_EXIT_XDTR_ACCESS:
9743 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9744 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9745 {
9746 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9747 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9748 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9749 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9750 }
9751 break;
9752
9753 case VMX_EXIT_TR_ACCESS:
9754 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9755 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9756 {
9757 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9758 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9759 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9760 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9761 }
9762 break;
9763
9764 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9765 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9766 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9767 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9768 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9769 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9770 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9771 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9772 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9773 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9774 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9775
9776 /* Events that aren't relevant at this point. */
9777 case VMX_EXIT_EXT_INT:
9778 case VMX_EXIT_INT_WINDOW:
9779 case VMX_EXIT_NMI_WINDOW:
9780 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9781 case VMX_EXIT_PREEMPT_TIMER:
9782 case VMX_EXIT_IO_INSTR:
9783 break;
9784
9785 /* Errors and unexpected events. */
9786 case VMX_EXIT_INIT_SIGNAL:
9787 case VMX_EXIT_SIPI:
9788 case VMX_EXIT_IO_SMI:
9789 case VMX_EXIT_SMI:
9790 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9791 case VMX_EXIT_ERR_MSR_LOAD:
9792 case VMX_EXIT_ERR_MACHINE_CHECK:
9793 break;
9794
9795 default:
9796 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9797 break;
9798 }
9799#undef SET_BOTH
9800#undef SET_EXIT
9801
9802 /*
9803 * Dtrace tracepoints go first. We do them here at once so we don't
9804 * have to copy the guest state saving and stuff a few dozen times.
9805 * Down side is that we've got to repeat the switch, though this time
9806 * we use enmEvent since the probes are a subset of what DBGF does.
9807 */
9808 if (fDtrace1 || fDtrace2)
9809 {
9810 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9811 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9812 switch (enmEvent1)
9813 {
9814 /** @todo consider which extra parameters would be helpful for each probe. */
9815 case DBGFEVENT_END: break;
9816 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9818 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9823 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9824 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9825 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9826 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9827 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9828 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9829 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9833 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9834 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9835 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9836 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9839 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9842 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9843 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9844 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9845 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9846 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9847 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9848 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9849 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9850 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9880 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9881 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9882 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9883 }
9884 switch (enmEvent2)
9885 {
9886 /** @todo consider which extra parameters would be helpful for each probe. */
9887 case DBGFEVENT_END: break;
9888 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9890 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9896 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9897 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9898 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9899 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9900 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9901 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9902 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9903 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9904 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9905 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9906 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9908 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9929 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9930 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9931 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9932 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9933 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9934 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9935 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9936 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9937 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9938 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9939 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9940 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9941 }
9942 }
9943
9944 /*
9945 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9946 * the DBGF call will do a full check).
9947 *
9948 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9949 * Note! If we have to events, we prioritize the first, i.e. the instruction
9950 * one, in order to avoid event nesting.
9951 */
9952 if ( enmEvent1 != DBGFEVENT_END
9953 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9954 {
9955 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9956 if (rcStrict != VINF_SUCCESS)
9957 return rcStrict;
9958 }
9959 else if ( enmEvent2 != DBGFEVENT_END
9960 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9961 {
9962 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9963 if (rcStrict != VINF_SUCCESS)
9964 return rcStrict;
9965 }
9966
9967 return VINF_SUCCESS;
9968}
9969
9970
9971/**
9972 * Single-stepping VM-exit filtering.
9973 *
9974 * This is preprocessing the VM-exits and deciding whether we've gotten far
9975 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9976 * handling is performed.
9977 *
9978 * @returns Strict VBox status code (i.e. informational status codes too).
9979 * @param pVM The cross context VM structure.
9980 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9981 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9982 * out-of-sync. Make sure to update the required
9983 * fields before using them.
9984 * @param pVmxTransient Pointer to the VMX-transient structure.
9985 * @param uExitReason The VM-exit reason.
9986 * @param pDbgState The debug state.
9987 */
9988DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9989 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9990{
9991 /*
9992 * Expensive (saves context) generic dtrace VM-exit probe.
9993 */
9994 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9995 { /* more likely */ }
9996 else
9997 {
9998 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9999 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10000 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10001 }
10002
10003 /*
10004 * Check for host NMI, just to get that out of the way.
10005 */
10006 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10007 { /* normally likely */ }
10008 else
10009 {
10010 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10011 AssertRCReturn(rc2, rc2);
10012 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10013 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10014 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10015 }
10016
10017 /*
10018 * Check for single stepping event if we're stepping.
10019 */
10020 if (pVCpu->hm.s.fSingleInstruction)
10021 {
10022 switch (uExitReason)
10023 {
10024 case VMX_EXIT_MTF:
10025 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10026
10027 /* Various events: */
10028 case VMX_EXIT_XCPT_OR_NMI:
10029 case VMX_EXIT_EXT_INT:
10030 case VMX_EXIT_TRIPLE_FAULT:
10031 case VMX_EXIT_INT_WINDOW:
10032 case VMX_EXIT_NMI_WINDOW:
10033 case VMX_EXIT_TASK_SWITCH:
10034 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10035 case VMX_EXIT_APIC_ACCESS:
10036 case VMX_EXIT_EPT_VIOLATION:
10037 case VMX_EXIT_EPT_MISCONFIG:
10038 case VMX_EXIT_PREEMPT_TIMER:
10039
10040 /* Instruction specific VM-exits: */
10041 case VMX_EXIT_CPUID:
10042 case VMX_EXIT_GETSEC:
10043 case VMX_EXIT_HLT:
10044 case VMX_EXIT_INVD:
10045 case VMX_EXIT_INVLPG:
10046 case VMX_EXIT_RDPMC:
10047 case VMX_EXIT_RDTSC:
10048 case VMX_EXIT_RSM:
10049 case VMX_EXIT_VMCALL:
10050 case VMX_EXIT_VMCLEAR:
10051 case VMX_EXIT_VMLAUNCH:
10052 case VMX_EXIT_VMPTRLD:
10053 case VMX_EXIT_VMPTRST:
10054 case VMX_EXIT_VMREAD:
10055 case VMX_EXIT_VMRESUME:
10056 case VMX_EXIT_VMWRITE:
10057 case VMX_EXIT_VMXOFF:
10058 case VMX_EXIT_VMXON:
10059 case VMX_EXIT_MOV_CRX:
10060 case VMX_EXIT_MOV_DRX:
10061 case VMX_EXIT_IO_INSTR:
10062 case VMX_EXIT_RDMSR:
10063 case VMX_EXIT_WRMSR:
10064 case VMX_EXIT_MWAIT:
10065 case VMX_EXIT_MONITOR:
10066 case VMX_EXIT_PAUSE:
10067 case VMX_EXIT_XDTR_ACCESS:
10068 case VMX_EXIT_TR_ACCESS:
10069 case VMX_EXIT_INVEPT:
10070 case VMX_EXIT_RDTSCP:
10071 case VMX_EXIT_INVVPID:
10072 case VMX_EXIT_WBINVD:
10073 case VMX_EXIT_XSETBV:
10074 case VMX_EXIT_RDRAND:
10075 case VMX_EXIT_INVPCID:
10076 case VMX_EXIT_VMFUNC:
10077 case VMX_EXIT_RDSEED:
10078 case VMX_EXIT_XSAVES:
10079 case VMX_EXIT_XRSTORS:
10080 {
10081 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10082 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10083 AssertRCReturn(rc2, rc2);
10084 if ( pMixedCtx->rip != pDbgState->uRipStart
10085 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10086 return VINF_EM_DBG_STEPPED;
10087 break;
10088 }
10089
10090 /* Errors and unexpected events: */
10091 case VMX_EXIT_INIT_SIGNAL:
10092 case VMX_EXIT_SIPI:
10093 case VMX_EXIT_IO_SMI:
10094 case VMX_EXIT_SMI:
10095 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10096 case VMX_EXIT_ERR_MSR_LOAD:
10097 case VMX_EXIT_ERR_MACHINE_CHECK:
10098 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10099 break;
10100
10101 default:
10102 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10103 break;
10104 }
10105 }
10106
10107 /*
10108 * Check for debugger event breakpoints and dtrace probes.
10109 */
10110 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10111 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10112 {
10113 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10114 if (rcStrict != VINF_SUCCESS)
10115 return rcStrict;
10116 }
10117
10118 /*
10119 * Normal processing.
10120 */
10121#ifdef HMVMX_USE_FUNCTION_TABLE
10122 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10123#else
10124 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10125#endif
10126}
10127
10128
10129/**
10130 * Single steps guest code using VT-x.
10131 *
10132 * @returns Strict VBox status code (i.e. informational status codes too).
10133 * @param pVM The cross context VM structure.
10134 * @param pVCpu The cross context virtual CPU structure.
10135 * @param pCtx Pointer to the guest-CPU context.
10136 *
10137 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10138 */
10139static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10140{
10141 VMXTRANSIENT VmxTransient;
10142 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10143
10144 /* Set HMCPU indicators. */
10145 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10146 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10147 pVCpu->hm.s.fDebugWantRdTscExit = false;
10148 pVCpu->hm.s.fUsingDebugLoop = true;
10149
10150 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10151 VMXRUNDBGSTATE DbgState;
10152 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10153 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10154
10155 /*
10156 * The loop.
10157 */
10158 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10159 for (uint32_t cLoops = 0; ; cLoops++)
10160 {
10161 Assert(!HMR0SuspendPending());
10162 HMVMX_ASSERT_CPU_SAFE();
10163 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10164
10165 /*
10166 * Preparatory work for running guest code, this may force us to return
10167 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10168 */
10169 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10170 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10171 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10172 if (rcStrict != VINF_SUCCESS)
10173 break;
10174
10175 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10176 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10177
10178 /*
10179 * Now we can run the guest code.
10180 */
10181 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10182
10183 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10184
10185 /*
10186 * Restore any residual host-state and save any bits shared between host
10187 * and guest into the guest-CPU state. Re-enables interrupts!
10188 */
10189 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10190
10191 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10192 if (RT_SUCCESS(rcRun))
10193 { /* very likely */ }
10194 else
10195 {
10196 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10197 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10198 return rcRun;
10199 }
10200
10201 /* Profile the VM-exit. */
10202 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10203 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10204 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10205 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10206 HMVMX_START_EXIT_DISPATCH_PROF();
10207
10208 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10209
10210 /*
10211 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10212 */
10213 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10214 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10215 if (rcStrict != VINF_SUCCESS)
10216 break;
10217 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10218 {
10219 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10220 rcStrict = VINF_EM_RAW_INTERRUPT;
10221 break;
10222 }
10223
10224 /*
10225 * Stepping: Did the RIP change, if so, consider it a single step.
10226 * Otherwise, make sure one of the TFs gets set.
10227 */
10228 if (fStepping)
10229 {
10230 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10231 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10232 AssertRCReturn(rc2, rc2);
10233 if ( pCtx->rip != DbgState.uRipStart
10234 || pCtx->cs.Sel != DbgState.uCsStart)
10235 {
10236 rcStrict = VINF_EM_DBG_STEPPED;
10237 break;
10238 }
10239 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10240 }
10241
10242 /*
10243 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10244 */
10245 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10246 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10247 }
10248
10249 /*
10250 * Clear the X86_EFL_TF if necessary.
10251 */
10252 if (pVCpu->hm.s.fClearTrapFlag)
10253 {
10254 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10255 AssertRCReturn(rc2, rc2);
10256 pVCpu->hm.s.fClearTrapFlag = false;
10257 pCtx->eflags.Bits.u1TF = 0;
10258 }
10259 /** @todo there seems to be issues with the resume flag when the monitor trap
10260 * flag is pending without being used. Seen early in bios init when
10261 * accessing APIC page in protected mode. */
10262
10263 /*
10264 * Restore VM-exit control settings as we may not reenter this function the
10265 * next time around.
10266 */
10267 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10268
10269 /* Restore HMCPU indicators. */
10270 pVCpu->hm.s.fUsingDebugLoop = false;
10271 pVCpu->hm.s.fDebugWantRdTscExit = false;
10272 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10273
10274 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10275 return rcStrict;
10276}
10277
10278
10279/** @} */
10280
10281
10282/**
10283 * Checks if any expensive dtrace probes are enabled and we should go to the
10284 * debug loop.
10285 *
10286 * @returns true if we should use debug loop, false if not.
10287 */
10288static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10289{
10290 /* It's probably faster to OR the raw 32-bit counter variables together.
10291 Since the variables are in an array and the probes are next to one
10292 another (more or less), we have good locality. So, better read
10293 eight-nine cache lines ever time and only have one conditional, than
10294 128+ conditionals, right? */
10295 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10296 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10297 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10298 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10299 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10300 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10301 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10302 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10303 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10304 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10305 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10306 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10307 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10308 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10309 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10310 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10311 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10312 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10313 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10314 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10315 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10316 ) != 0
10317 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10318 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10319 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10320 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10321 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10322 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10323 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10324 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10325 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10326 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10327 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10328 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10329 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10330 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10331 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10332 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10333 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10334 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10335 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10336 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10337 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10338 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10339 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10340 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10341 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10342 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10343 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10344 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10345 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10346 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10347 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10348 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10349 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10350 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10351 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10352 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10353 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10354 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10355 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10356 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10357 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10358 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10359 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10360 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10361 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10362 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10363 ) != 0
10364 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10365 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10366 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10367 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10368 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10369 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10370 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10371 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10372 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10373 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10374 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10375 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10376 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10377 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10378 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10379 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10380 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10381 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10382 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10383 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10384 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10385 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10386 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10387 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10388 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10389 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10390 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10391 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10392 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10393 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10394 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10395 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10396 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10397 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10398 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10399 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10400 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10401 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10402 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10403 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10404 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10405 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10406 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10407 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10408 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10409 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10410 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10411 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10412 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10413 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10414 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10415 ) != 0;
10416}
10417
10418
10419/**
10420 * Runs the guest code using VT-x.
10421 *
10422 * @returns Strict VBox status code (i.e. informational status codes too).
10423 * @param pVM The cross context VM structure.
10424 * @param pVCpu The cross context virtual CPU structure.
10425 * @param pCtx Pointer to the guest-CPU context.
10426 */
10427VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10428{
10429 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10430 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10431 HMVMX_ASSERT_PREEMPT_SAFE();
10432
10433 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10434
10435 VBOXSTRICTRC rcStrict;
10436 if ( !pVCpu->hm.s.fUseDebugLoop
10437 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10438 && !DBGFIsStepping(pVCpu)
10439 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10440 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10441 else
10442 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10443
10444 if (rcStrict == VERR_EM_INTERPRETER)
10445 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10446 else if (rcStrict == VINF_EM_RESET)
10447 rcStrict = VINF_EM_TRIPLE_FAULT;
10448
10449 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10450 if (RT_FAILURE(rc2))
10451 {
10452 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10453 rcStrict = rc2;
10454 }
10455 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10456 return rcStrict;
10457}
10458
10459
10460#ifndef HMVMX_USE_FUNCTION_TABLE
10461DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10462{
10463# ifdef DEBUG_ramshankar
10464# define RETURN_EXIT_CALL(a_CallExpr) \
10465 do { \
10466 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10467 VBOXSTRICTRC rcStrict = a_CallExpr; \
10468 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10469 return rcStrict; \
10470 } while (0)
10471# else
10472# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10473# endif
10474 switch (rcReason)
10475 {
10476 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10477 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10478 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10479 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10480 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10481 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10494 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10499 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10500 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10501 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10502 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10503 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10504 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10505 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10506 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10507 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10508 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10509 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10510
10511 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10512 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10513 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10514 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10515 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10516 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10517 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10518 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10519 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10520
10521 case VMX_EXIT_VMCLEAR:
10522 case VMX_EXIT_VMLAUNCH:
10523 case VMX_EXIT_VMPTRLD:
10524 case VMX_EXIT_VMPTRST:
10525 case VMX_EXIT_VMREAD:
10526 case VMX_EXIT_VMRESUME:
10527 case VMX_EXIT_VMWRITE:
10528 case VMX_EXIT_VMXOFF:
10529 case VMX_EXIT_VMXON:
10530 case VMX_EXIT_INVEPT:
10531 case VMX_EXIT_INVVPID:
10532 case VMX_EXIT_VMFUNC:
10533 case VMX_EXIT_XSAVES:
10534 case VMX_EXIT_XRSTORS:
10535 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10536 case VMX_EXIT_ENCLS:
10537 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10538 case VMX_EXIT_PML_FULL:
10539 default:
10540 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10541 }
10542#undef RETURN_EXIT_CALL
10543}
10544#endif /* !HMVMX_USE_FUNCTION_TABLE */
10545
10546
10547#ifdef VBOX_STRICT
10548/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10549# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10550 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10551
10552# define HMVMX_ASSERT_PREEMPT_CPUID() \
10553 do { \
10554 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10555 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10556 } while (0)
10557
10558# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10559 do { \
10560 AssertPtr(pVCpu); \
10561 AssertPtr(pMixedCtx); \
10562 AssertPtr(pVmxTransient); \
10563 Assert(pVmxTransient->fVMEntryFailed == false); \
10564 Assert(ASMIntAreEnabled()); \
10565 HMVMX_ASSERT_PREEMPT_SAFE(); \
10566 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10567 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10568 HMVMX_ASSERT_PREEMPT_SAFE(); \
10569 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10570 HMVMX_ASSERT_PREEMPT_CPUID(); \
10571 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10572 } while (0)
10573
10574# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10575 do { \
10576 Log4Func(("\n")); \
10577 } while (0)
10578#else /* nonstrict builds: */
10579# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10580 do { \
10581 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10582 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10583 } while (0)
10584# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10585#endif
10586
10587
10588/**
10589 * Advances the guest RIP by the specified number of bytes.
10590 *
10591 * @param pVCpu The cross context virtual CPU structure.
10592 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10593 * out-of-sync. Make sure to update the required fields
10594 * before using them.
10595 * @param cbInstr Number of bytes to advance the RIP by.
10596 *
10597 * @remarks No-long-jump zone!!!
10598 */
10599DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10600{
10601 /* Advance the RIP. */
10602 pMixedCtx->rip += cbInstr;
10603 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10604
10605 /* Update interrupt inhibition. */
10606 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10607 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10608 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10609}
10610
10611
10612/**
10613 * Advances the guest RIP after reading it from the VMCS.
10614 *
10615 * @returns VBox status code, no informational status codes.
10616 * @param pVCpu The cross context virtual CPU structure.
10617 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10618 * out-of-sync. Make sure to update the required fields
10619 * before using them.
10620 * @param pVmxTransient Pointer to the VMX transient structure.
10621 *
10622 * @remarks No-long-jump zone!!!
10623 */
10624static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10625{
10626 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10627 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10628 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10629 AssertRCReturn(rc, rc);
10630
10631 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10632
10633 /*
10634 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10635 * pending debug exception field as it takes care of priority of events.
10636 *
10637 * See Intel spec. 32.2.1 "Debug Exceptions".
10638 */
10639 if ( !pVCpu->hm.s.fSingleInstruction
10640 && pMixedCtx->eflags.Bits.u1TF)
10641 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10642
10643 return VINF_SUCCESS;
10644}
10645
10646
10647/**
10648 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10649 * and update error record fields accordingly.
10650 *
10651 * @return VMX_IGS_* return codes.
10652 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10653 * wrong with the guest state.
10654 *
10655 * @param pVM The cross context VM structure.
10656 * @param pVCpu The cross context virtual CPU structure.
10657 * @param pCtx Pointer to the guest-CPU state.
10658 *
10659 * @remarks This function assumes our cache of the VMCS controls
10660 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10661 */
10662static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10663{
10664#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10665#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10666 uError = (err); \
10667 break; \
10668 } else do { } while (0)
10669
10670 int rc;
10671 uint32_t uError = VMX_IGS_ERROR;
10672 uint32_t u32Val;
10673 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10674
10675 do
10676 {
10677 /*
10678 * CR0.
10679 */
10680 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10681 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10682 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10683 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10684 if (fUnrestrictedGuest)
10685 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10686
10687 uint32_t u32GuestCR0;
10688 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10689 AssertRCBreak(rc);
10690 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10691 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10692 if ( !fUnrestrictedGuest
10693 && (u32GuestCR0 & X86_CR0_PG)
10694 && !(u32GuestCR0 & X86_CR0_PE))
10695 {
10696 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10697 }
10698
10699 /*
10700 * CR4.
10701 */
10702 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10703 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10704
10705 uint32_t u32GuestCR4;
10706 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10707 AssertRCBreak(rc);
10708 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10709 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10710
10711 /*
10712 * IA32_DEBUGCTL MSR.
10713 */
10714 uint64_t u64Val;
10715 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10716 AssertRCBreak(rc);
10717 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10718 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10719 {
10720 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10721 }
10722 uint64_t u64DebugCtlMsr = u64Val;
10723
10724#ifdef VBOX_STRICT
10725 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10726 AssertRCBreak(rc);
10727 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10728#endif
10729 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10730
10731 /*
10732 * RIP and RFLAGS.
10733 */
10734 uint32_t u32Eflags;
10735#if HC_ARCH_BITS == 64
10736 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10737 AssertRCBreak(rc);
10738 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10739 if ( !fLongModeGuest
10740 || !pCtx->cs.Attr.n.u1Long)
10741 {
10742 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10743 }
10744 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10745 * must be identical if the "IA-32e mode guest" VM-entry
10746 * control is 1 and CS.L is 1. No check applies if the
10747 * CPU supports 64 linear-address bits. */
10748
10749 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10750 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10751 AssertRCBreak(rc);
10752 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10753 VMX_IGS_RFLAGS_RESERVED);
10754 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10755 u32Eflags = u64Val;
10756#else
10757 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10758 AssertRCBreak(rc);
10759 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10760 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10761#endif
10762
10763 if ( fLongModeGuest
10764 || ( fUnrestrictedGuest
10765 && !(u32GuestCR0 & X86_CR0_PE)))
10766 {
10767 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10768 }
10769
10770 uint32_t u32EntryInfo;
10771 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10772 AssertRCBreak(rc);
10773 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10774 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10775 {
10776 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10777 }
10778
10779 /*
10780 * 64-bit checks.
10781 */
10782#if HC_ARCH_BITS == 64
10783 if (fLongModeGuest)
10784 {
10785 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10786 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10787 }
10788
10789 if ( !fLongModeGuest
10790 && (u32GuestCR4 & X86_CR4_PCIDE))
10791 {
10792 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10793 }
10794
10795 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10796 * 51:32 beyond the processor's physical-address width are 0. */
10797
10798 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10799 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10800 {
10801 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10802 }
10803
10804 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10805 AssertRCBreak(rc);
10806 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10807
10808 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10809 AssertRCBreak(rc);
10810 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10811#endif
10812
10813 /*
10814 * PERF_GLOBAL MSR.
10815 */
10816 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10817 {
10818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10821 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10822 }
10823
10824 /*
10825 * PAT MSR.
10826 */
10827 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10828 {
10829 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10830 AssertRCBreak(rc);
10831 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10832 for (unsigned i = 0; i < 8; i++)
10833 {
10834 uint8_t u8Val = (u64Val & 0xff);
10835 if ( u8Val != 0 /* UC */
10836 && u8Val != 1 /* WC */
10837 && u8Val != 4 /* WT */
10838 && u8Val != 5 /* WP */
10839 && u8Val != 6 /* WB */
10840 && u8Val != 7 /* UC- */)
10841 {
10842 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10843 }
10844 u64Val >>= 8;
10845 }
10846 }
10847
10848 /*
10849 * EFER MSR.
10850 */
10851 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10852 {
10853 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10854 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10855 AssertRCBreak(rc);
10856 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10857 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10858 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10859 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10860 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10861 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10862 || !(u32GuestCR0 & X86_CR0_PG)
10863 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10864 VMX_IGS_EFER_LMA_LME_MISMATCH);
10865 }
10866
10867 /*
10868 * Segment registers.
10869 */
10870 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10871 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10872 if (!(u32Eflags & X86_EFL_VM))
10873 {
10874 /* CS */
10875 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10876 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10877 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10878 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10879 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10880 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10881 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10882 /* CS cannot be loaded with NULL in protected mode. */
10883 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10884 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10885 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10886 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10887 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10888 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10889 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10890 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10891 else
10892 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10893
10894 /* SS */
10895 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10896 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10897 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10898 if ( !(pCtx->cr0 & X86_CR0_PE)
10899 || pCtx->cs.Attr.n.u4Type == 3)
10900 {
10901 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10902 }
10903 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10904 {
10905 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10906 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10907 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10908 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10909 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10910 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10911 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10912 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10913 }
10914
10915 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10916 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10917 {
10918 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10919 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10920 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10921 || pCtx->ds.Attr.n.u4Type > 11
10922 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10923 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10924 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10925 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10926 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10927 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10928 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10929 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10930 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10931 }
10932 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10933 {
10934 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10935 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10936 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10937 || pCtx->es.Attr.n.u4Type > 11
10938 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10939 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10940 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10941 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10942 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10943 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10944 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10945 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10946 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10947 }
10948 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10949 {
10950 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10951 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10952 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10953 || pCtx->fs.Attr.n.u4Type > 11
10954 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10955 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10956 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10957 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10958 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10959 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10960 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10961 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10962 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10963 }
10964 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10965 {
10966 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10967 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10968 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10969 || pCtx->gs.Attr.n.u4Type > 11
10970 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10971 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10972 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10973 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10974 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10975 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10976 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10977 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10978 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10979 }
10980 /* 64-bit capable CPUs. */
10981#if HC_ARCH_BITS == 64
10982 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10983 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10984 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10985 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10986 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10987 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10988 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10989 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10990 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10991 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10992 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10993#endif
10994 }
10995 else
10996 {
10997 /* V86 mode checks. */
10998 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10999 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11000 {
11001 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11002 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11003 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11004 }
11005 else
11006 {
11007 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11008 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11009 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11010 }
11011
11012 /* CS */
11013 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11014 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11015 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11016 /* SS */
11017 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11018 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11019 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11020 /* DS */
11021 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11022 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11023 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11024 /* ES */
11025 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11026 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11027 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11028 /* FS */
11029 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11030 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11031 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11032 /* GS */
11033 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11034 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11035 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11036 /* 64-bit capable CPUs. */
11037#if HC_ARCH_BITS == 64
11038 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11039 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11040 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11041 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11042 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11043 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11044 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11045 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11046 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11047 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11048 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11049#endif
11050 }
11051
11052 /*
11053 * TR.
11054 */
11055 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11056 /* 64-bit capable CPUs. */
11057#if HC_ARCH_BITS == 64
11058 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11059#endif
11060 if (fLongModeGuest)
11061 {
11062 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11063 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11064 }
11065 else
11066 {
11067 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11068 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11069 VMX_IGS_TR_ATTR_TYPE_INVALID);
11070 }
11071 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11072 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11073 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11074 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11075 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11076 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11077 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11078 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11079
11080 /*
11081 * GDTR and IDTR.
11082 */
11083#if HC_ARCH_BITS == 64
11084 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11085 AssertRCBreak(rc);
11086 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11087
11088 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11089 AssertRCBreak(rc);
11090 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11091#endif
11092
11093 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11094 AssertRCBreak(rc);
11095 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11096
11097 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11098 AssertRCBreak(rc);
11099 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11100
11101 /*
11102 * Guest Non-Register State.
11103 */
11104 /* Activity State. */
11105 uint32_t u32ActivityState;
11106 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11107 AssertRCBreak(rc);
11108 HMVMX_CHECK_BREAK( !u32ActivityState
11109 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11110 VMX_IGS_ACTIVITY_STATE_INVALID);
11111 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11112 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11113 uint32_t u32IntrState;
11114 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11115 AssertRCBreak(rc);
11116 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11117 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11118 {
11119 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11120 }
11121
11122 /** @todo Activity state and injecting interrupts. Left as a todo since we
11123 * currently don't use activity states but ACTIVE. */
11124
11125 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11126 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11127
11128 /* Guest interruptibility-state. */
11129 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11130 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11131 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11132 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11133 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11134 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11135 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11136 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11137 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11138 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11139 {
11140 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11141 {
11142 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11143 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11144 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11145 }
11146 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11147 {
11148 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11149 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11150 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11151 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11152 }
11153 }
11154 /** @todo Assumes the processor is not in SMM. */
11155 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11156 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11157 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11158 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11159 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11160 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11161 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11162 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11163 {
11164 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11165 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11166 }
11167
11168 /* Pending debug exceptions. */
11169#if HC_ARCH_BITS == 64
11170 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11171 AssertRCBreak(rc);
11172 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11173 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11174 u32Val = u64Val; /* For pending debug exceptions checks below. */
11175#else
11176 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11177 AssertRCBreak(rc);
11178 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11179 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11180#endif
11181
11182 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11183 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11184 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11185 {
11186 if ( (u32Eflags & X86_EFL_TF)
11187 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11188 {
11189 /* Bit 14 is PendingDebug.BS. */
11190 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11191 }
11192 if ( !(u32Eflags & X86_EFL_TF)
11193 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11194 {
11195 /* Bit 14 is PendingDebug.BS. */
11196 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11197 }
11198 }
11199
11200 /* VMCS link pointer. */
11201 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11202 AssertRCBreak(rc);
11203 if (u64Val != UINT64_C(0xffffffffffffffff))
11204 {
11205 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11206 /** @todo Bits beyond the processor's physical-address width MBZ. */
11207 /** @todo 32-bit located in memory referenced by value of this field (as a
11208 * physical address) must contain the processor's VMCS revision ID. */
11209 /** @todo SMM checks. */
11210 }
11211
11212 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11213 * not using Nested Paging? */
11214 if ( pVM->hm.s.fNestedPaging
11215 && !fLongModeGuest
11216 && CPUMIsGuestInPAEModeEx(pCtx))
11217 {
11218 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11219 AssertRCBreak(rc);
11220 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11221
11222 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11223 AssertRCBreak(rc);
11224 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11225
11226 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11227 AssertRCBreak(rc);
11228 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11229
11230 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11231 AssertRCBreak(rc);
11232 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11233 }
11234
11235 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11236 if (uError == VMX_IGS_ERROR)
11237 uError = VMX_IGS_REASON_NOT_FOUND;
11238 } while (0);
11239
11240 pVCpu->hm.s.u32HMError = uError;
11241 return uError;
11242
11243#undef HMVMX_ERROR_BREAK
11244#undef HMVMX_CHECK_BREAK
11245}
11246
11247/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11248/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11249/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11250
11251/** @name VM-exit handlers.
11252 * @{
11253 */
11254
11255/**
11256 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11257 */
11258HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11259{
11260 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11261 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11262 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11263 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11264 return VINF_SUCCESS;
11265 return VINF_EM_RAW_INTERRUPT;
11266}
11267
11268
11269/**
11270 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11271 */
11272HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11273{
11274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11275 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11276
11277 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11278 AssertRCReturn(rc, rc);
11279
11280 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11281 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11282 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11283 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11284
11285 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11286 {
11287 /*
11288 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11289 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11290 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11291 *
11292 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11293 */
11294 VMXDispatchHostNmi();
11295 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11296 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11297 return VINF_SUCCESS;
11298 }
11299
11300 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11301 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11302 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11303 { /* likely */ }
11304 else
11305 {
11306 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11307 rcStrictRc1 = VINF_SUCCESS;
11308 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11309 return rcStrictRc1;
11310 }
11311
11312 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11313 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11314 switch (uIntType)
11315 {
11316 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11317 Assert(uVector == X86_XCPT_DB);
11318 /* no break */
11319 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11320 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11321 /* no break */
11322 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11323 {
11324 /*
11325 * If there's any exception caused as a result of event injection, go back to
11326 * the interpreter. The page-fault case is complicated and we manually handle
11327 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11328 * handled in hmR0VmxCheckExitDueToEventDelivery.
11329 */
11330 if (!pVCpu->hm.s.Event.fPending)
11331 { /* likely */ }
11332 else if ( uVector != X86_XCPT_PF
11333 && uVector != X86_XCPT_AC)
11334 {
11335 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11336 rc = VERR_EM_INTERPRETER;
11337 break;
11338 }
11339
11340 switch (uVector)
11341 {
11342 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11343 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11344 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11345 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11346 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11347 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11348 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11349
11350 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11351 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11352 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11353 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11354 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11355 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11356 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11357 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11358 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11359 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11360 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11361 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11362 default:
11363 {
11364 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11365 AssertRCReturn(rc, rc);
11366
11367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11368 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11369 {
11370 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11371 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11372 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11373
11374 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11375 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11376 AssertRCReturn(rc, rc);
11377 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11378 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11379 0 /* GCPtrFaultAddress */);
11380 AssertRCReturn(rc, rc);
11381 }
11382 else
11383 {
11384 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11385 pVCpu->hm.s.u32HMError = uVector;
11386 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11387 }
11388 break;
11389 }
11390 }
11391 break;
11392 }
11393
11394 default:
11395 {
11396 pVCpu->hm.s.u32HMError = uExitIntInfo;
11397 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11398 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11399 break;
11400 }
11401 }
11402 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11403 return rc;
11404}
11405
11406
11407/**
11408 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11409 */
11410HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11411{
11412 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11413
11414 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11415 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11416
11417 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11419 return VINF_SUCCESS;
11420}
11421
11422
11423/**
11424 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11425 */
11426HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11427{
11428 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11429 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11430 {
11431 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11432 HMVMX_RETURN_UNEXPECTED_EXIT();
11433 }
11434
11435 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11436
11437 /*
11438 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11439 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11440 */
11441 uint32_t uIntrState = 0;
11442 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11443 AssertRCReturn(rc, rc);
11444
11445 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11446 if ( fBlockSti
11447 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11448 {
11449 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11450 }
11451
11452 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11453 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11454
11455 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11456 return VINF_SUCCESS;
11457}
11458
11459
11460/**
11461 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11462 */
11463HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11464{
11465 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11467 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11468}
11469
11470
11471/**
11472 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11473 */
11474HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11475{
11476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11478 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11479}
11480
11481
11482/**
11483 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11484 */
11485HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11486{
11487 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11488 PVM pVM = pVCpu->CTX_SUFF(pVM);
11489 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11490 if (RT_LIKELY(rc == VINF_SUCCESS))
11491 {
11492 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11493 Assert(pVmxTransient->cbInstr == 2);
11494 }
11495 else
11496 {
11497 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11498 rc = VERR_EM_INTERPRETER;
11499 }
11500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11501 return rc;
11502}
11503
11504
11505/**
11506 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11507 */
11508HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11509{
11510 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11511 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11512 AssertRCReturn(rc, rc);
11513
11514 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11515 return VINF_EM_RAW_EMULATE_INSTR;
11516
11517 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11518 HMVMX_RETURN_UNEXPECTED_EXIT();
11519}
11520
11521
11522/**
11523 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11524 */
11525HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11526{
11527 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11528 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11529 AssertRCReturn(rc, rc);
11530
11531 PVM pVM = pVCpu->CTX_SUFF(pVM);
11532 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11533 if (RT_LIKELY(rc == VINF_SUCCESS))
11534 {
11535 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11536 Assert(pVmxTransient->cbInstr == 2);
11537 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11538 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11539 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11540 }
11541 else
11542 rc = VERR_EM_INTERPRETER;
11543 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11544 return rc;
11545}
11546
11547
11548/**
11549 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11550 */
11551HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11552{
11553 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11554 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11555 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11556 AssertRCReturn(rc, rc);
11557
11558 PVM pVM = pVCpu->CTX_SUFF(pVM);
11559 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11560 if (RT_SUCCESS(rc))
11561 {
11562 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11563 Assert(pVmxTransient->cbInstr == 3);
11564 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11565 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11566 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11567 }
11568 else
11569 {
11570 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11571 rc = VERR_EM_INTERPRETER;
11572 }
11573 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11574 return rc;
11575}
11576
11577
11578/**
11579 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11580 */
11581HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11582{
11583 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11584 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11585 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11586 AssertRCReturn(rc, rc);
11587
11588 PVM pVM = pVCpu->CTX_SUFF(pVM);
11589 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11590 if (RT_LIKELY(rc == VINF_SUCCESS))
11591 {
11592 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11593 Assert(pVmxTransient->cbInstr == 2);
11594 }
11595 else
11596 {
11597 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11598 rc = VERR_EM_INTERPRETER;
11599 }
11600 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11601 return rc;
11602}
11603
11604
11605/**
11606 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11607 */
11608HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11609{
11610 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11611 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11612
11613 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11614 if (pVCpu->hm.s.fHypercallsEnabled)
11615 {
11616#if 0
11617 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11618#else
11619 /* Aggressive state sync. for now. */
11620 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11621 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11622 AssertRCReturn(rc, rc);
11623#endif
11624
11625 /* Perform the hypercall. */
11626 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11627 if (rcStrict == VINF_SUCCESS)
11628 {
11629 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11630 AssertRCReturn(rc, rc);
11631 }
11632 else
11633 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11634 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11635 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11636
11637 /* If the hypercall changes anything other than guest's general-purpose registers,
11638 we would need to reload the guest changed bits here before VM-entry. */
11639 }
11640 else
11641 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11642
11643 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11644 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11645 {
11646 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11647 rcStrict = VINF_SUCCESS;
11648 }
11649
11650 return rcStrict;
11651}
11652
11653
11654/**
11655 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11656 */
11657HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11658{
11659 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11660 PVM pVM = pVCpu->CTX_SUFF(pVM);
11661 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11662
11663 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11664 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11665 AssertRCReturn(rc, rc);
11666
11667 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11668 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11669 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11670 else
11671 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11672 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11673 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11674 return rcStrict;
11675}
11676
11677
11678/**
11679 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11680 */
11681HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11682{
11683 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11684 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11685 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11686 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11687 AssertRCReturn(rc, rc);
11688
11689 PVM pVM = pVCpu->CTX_SUFF(pVM);
11690 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11691 if (RT_LIKELY(rc == VINF_SUCCESS))
11692 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11693 else
11694 {
11695 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11696 rc = VERR_EM_INTERPRETER;
11697 }
11698 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11699 return rc;
11700}
11701
11702
11703/**
11704 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11705 */
11706HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11707{
11708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11709 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11710 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11711 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11712 AssertRCReturn(rc, rc);
11713
11714 PVM pVM = pVCpu->CTX_SUFF(pVM);
11715 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11716 rc = VBOXSTRICTRC_VAL(rc2);
11717 if (RT_LIKELY( rc == VINF_SUCCESS
11718 || rc == VINF_EM_HALT))
11719 {
11720 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11721 AssertRCReturn(rc3, rc3);
11722
11723 if ( rc == VINF_EM_HALT
11724 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11725 {
11726 rc = VINF_SUCCESS;
11727 }
11728 }
11729 else
11730 {
11731 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11732 rc = VERR_EM_INTERPRETER;
11733 }
11734 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11735 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11737 return rc;
11738}
11739
11740
11741/**
11742 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11743 */
11744HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11745{
11746 /*
11747 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11748 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11749 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11750 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11751 */
11752 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11753 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11754 HMVMX_RETURN_UNEXPECTED_EXIT();
11755}
11756
11757
11758/**
11759 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11760 */
11761HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11762{
11763 /*
11764 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11765 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11766 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11767 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11768 */
11769 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11770 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11771 HMVMX_RETURN_UNEXPECTED_EXIT();
11772}
11773
11774
11775/**
11776 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11777 */
11778HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11779{
11780 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11782 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11783 HMVMX_RETURN_UNEXPECTED_EXIT();
11784}
11785
11786
11787/**
11788 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11789 */
11790HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11791{
11792 /*
11793 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11794 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11795 * See Intel spec. 25.3 "Other Causes of VM-exits".
11796 */
11797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11798 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11799 HMVMX_RETURN_UNEXPECTED_EXIT();
11800}
11801
11802
11803/**
11804 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11805 * VM-exit.
11806 */
11807HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11808{
11809 /*
11810 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11811 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11812 *
11813 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11814 * See Intel spec. "23.8 Restrictions on VMX operation".
11815 */
11816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11817 return VINF_SUCCESS;
11818}
11819
11820
11821/**
11822 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11823 * VM-exit.
11824 */
11825HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11826{
11827 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11828 return VINF_EM_RESET;
11829}
11830
11831
11832/**
11833 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11834 */
11835HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11836{
11837 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11838 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11839
11840 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11841 AssertRCReturn(rc, rc);
11842
11843 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11844 rc = VINF_SUCCESS;
11845 else
11846 rc = VINF_EM_HALT;
11847
11848 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11849 if (rc != VINF_SUCCESS)
11850 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11851 return rc;
11852}
11853
11854
11855/**
11856 * VM-exit handler for instructions that result in a \#UD exception delivered to
11857 * the guest.
11858 */
11859HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11860{
11861 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11862 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11863 return VINF_SUCCESS;
11864}
11865
11866
11867/**
11868 * VM-exit handler for expiry of the VMX preemption timer.
11869 */
11870HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11871{
11872 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11873
11874 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11875 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11876
11877 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11878 PVM pVM = pVCpu->CTX_SUFF(pVM);
11879 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11880 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11881 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11882}
11883
11884
11885/**
11886 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11887 */
11888HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11889{
11890 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11891
11892 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11893 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11894 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11895 AssertRCReturn(rc, rc);
11896
11897 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11898 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11899
11900 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11901
11902 return rcStrict;
11903}
11904
11905
11906/**
11907 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11908 */
11909HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11910{
11911 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11912
11913 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11914 /** @todo implement EMInterpretInvpcid() */
11915 return VERR_EM_INTERPRETER;
11916}
11917
11918
11919/**
11920 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11921 * Error VM-exit.
11922 */
11923HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11924{
11925 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11926 AssertRCReturn(rc, rc);
11927
11928 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11929 AssertRCReturn(rc, rc);
11930
11931 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11932 NOREF(uInvalidReason);
11933
11934#ifdef VBOX_STRICT
11935 uint32_t uIntrState;
11936 RTHCUINTREG uHCReg;
11937 uint64_t u64Val;
11938 uint32_t u32Val;
11939
11940 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11941 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11942 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11943 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11944 AssertRCReturn(rc, rc);
11945
11946 Log4(("uInvalidReason %u\n", uInvalidReason));
11947 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11948 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11949 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11950 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11951
11952 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11953 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11954 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11955 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11956 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11957 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11958 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11959 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11960 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11961 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11962 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11963 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11964#else
11965 NOREF(pVmxTransient);
11966#endif
11967
11968 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11969 return VERR_VMX_INVALID_GUEST_STATE;
11970}
11971
11972
11973/**
11974 * VM-exit handler for VM-entry failure due to an MSR-load
11975 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11976 */
11977HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11978{
11979 NOREF(pVmxTransient);
11980 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11981 HMVMX_RETURN_UNEXPECTED_EXIT();
11982}
11983
11984
11985/**
11986 * VM-exit handler for VM-entry failure due to a machine-check event
11987 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11988 */
11989HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11990{
11991 NOREF(pVmxTransient);
11992 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11993 HMVMX_RETURN_UNEXPECTED_EXIT();
11994}
11995
11996
11997/**
11998 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11999 * theory.
12000 */
12001HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12002{
12003 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12004 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12005 return VERR_VMX_UNDEFINED_EXIT_CODE;
12006}
12007
12008
12009/**
12010 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12011 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12012 * Conditional VM-exit.
12013 */
12014HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12015{
12016 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12017
12018 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12019 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12020 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12021 return VERR_EM_INTERPRETER;
12022 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12023 HMVMX_RETURN_UNEXPECTED_EXIT();
12024}
12025
12026
12027/**
12028 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12029 */
12030HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12031{
12032 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12033
12034 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12035 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12036 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12037 return VERR_EM_INTERPRETER;
12038 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12039 HMVMX_RETURN_UNEXPECTED_EXIT();
12040}
12041
12042
12043/**
12044 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12045 */
12046HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12047{
12048 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12049
12050 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12051 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12052 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12053 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12054 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12055 {
12056 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12057 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12058 }
12059 AssertRCReturn(rc, rc);
12060 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12061
12062#ifdef VBOX_STRICT
12063 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12064 {
12065 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12066 && pMixedCtx->ecx != MSR_K6_EFER)
12067 {
12068 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12069 pMixedCtx->ecx));
12070 HMVMX_RETURN_UNEXPECTED_EXIT();
12071 }
12072 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12073 {
12074 VMXMSREXITREAD enmRead;
12075 VMXMSREXITWRITE enmWrite;
12076 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12077 AssertRCReturn(rc2, rc2);
12078 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12079 {
12080 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12081 HMVMX_RETURN_UNEXPECTED_EXIT();
12082 }
12083 }
12084 }
12085#endif
12086
12087 PVM pVM = pVCpu->CTX_SUFF(pVM);
12088 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12089 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12090 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12092 if (RT_SUCCESS(rc))
12093 {
12094 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12095 Assert(pVmxTransient->cbInstr == 2);
12096 }
12097 return rc;
12098}
12099
12100
12101/**
12102 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12103 */
12104HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12105{
12106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12107 PVM pVM = pVCpu->CTX_SUFF(pVM);
12108 int rc = VINF_SUCCESS;
12109
12110 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12111 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12112 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12113 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12114 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12115 {
12116 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12117 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12118 }
12119 AssertRCReturn(rc, rc);
12120 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12121
12122 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12123 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12125
12126 if (RT_SUCCESS(rc))
12127 {
12128 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12129
12130 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12131 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12132 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12133 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12134 {
12135 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12136 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12137 EMInterpretWrmsr() changes it. */
12138 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12139 }
12140 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12141 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12142 else if (pMixedCtx->ecx == MSR_K6_EFER)
12143 {
12144 /*
12145 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12146 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12147 * the other bits as well, SCE and NXE. See @bugref{7368}.
12148 */
12149 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12150 }
12151
12152 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12153 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12154 {
12155 switch (pMixedCtx->ecx)
12156 {
12157 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12158 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12159 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12160 case MSR_K8_FS_BASE: /* no break */
12161 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12162 case MSR_K6_EFER: /* already handled above */ break;
12163 default:
12164 {
12165 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12166 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12167 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12168 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12169 break;
12170 }
12171 }
12172 }
12173#ifdef VBOX_STRICT
12174 else
12175 {
12176 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12177 switch (pMixedCtx->ecx)
12178 {
12179 case MSR_IA32_SYSENTER_CS:
12180 case MSR_IA32_SYSENTER_EIP:
12181 case MSR_IA32_SYSENTER_ESP:
12182 case MSR_K8_FS_BASE:
12183 case MSR_K8_GS_BASE:
12184 {
12185 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12186 HMVMX_RETURN_UNEXPECTED_EXIT();
12187 }
12188
12189 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12190 default:
12191 {
12192 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12193 {
12194 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12195 if (pMixedCtx->ecx != MSR_K6_EFER)
12196 {
12197 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12198 pMixedCtx->ecx));
12199 HMVMX_RETURN_UNEXPECTED_EXIT();
12200 }
12201 }
12202
12203 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12204 {
12205 VMXMSREXITREAD enmRead;
12206 VMXMSREXITWRITE enmWrite;
12207 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12208 AssertRCReturn(rc2, rc2);
12209 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12210 {
12211 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12212 HMVMX_RETURN_UNEXPECTED_EXIT();
12213 }
12214 }
12215 break;
12216 }
12217 }
12218 }
12219#endif /* VBOX_STRICT */
12220 }
12221 return rc;
12222}
12223
12224
12225/**
12226 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12227 */
12228HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12229{
12230 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12231
12232 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12233 return VINF_EM_RAW_INTERRUPT;
12234}
12235
12236
12237/**
12238 * VM-exit handler for when the TPR value is lowered below the specified
12239 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12240 */
12241HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12242{
12243 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12244 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12245
12246 /*
12247 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12248 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12249 */
12250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12251 return VINF_SUCCESS;
12252}
12253
12254
12255/**
12256 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12257 * VM-exit.
12258 *
12259 * @retval VINF_SUCCESS when guest execution can continue.
12260 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12261 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12262 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12263 * interpreter.
12264 */
12265HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12266{
12267 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12268 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12269 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12270 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12271 AssertRCReturn(rc, rc);
12272
12273 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12274 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12275 PVM pVM = pVCpu->CTX_SUFF(pVM);
12276 VBOXSTRICTRC rcStrict;
12277 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12278 switch (uAccessType)
12279 {
12280 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12281 {
12282 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12283 AssertRCReturn(rc, rc);
12284
12285 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12286 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12287 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12288 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12289 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12290 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12291 {
12292 case 0: /* CR0 */
12293 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12294 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12295 break;
12296 case 2: /* CR2 */
12297 /* Nothing to do here, CR2 it's not part of the VMCS. */
12298 break;
12299 case 3: /* CR3 */
12300 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12301 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12302 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12303 break;
12304 case 4: /* CR4 */
12305 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12306 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12307 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12308 break;
12309 case 8: /* CR8 */
12310 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12311 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12312 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12313 break;
12314 default:
12315 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12316 break;
12317 }
12318
12319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12320 break;
12321 }
12322
12323 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12324 {
12325 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12326 AssertRCReturn(rc, rc);
12327
12328 Assert( !pVM->hm.s.fNestedPaging
12329 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12330 || pVCpu->hm.s.fUsingDebugLoop
12331 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12332
12333 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12334 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12335 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12336
12337 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12338 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12339 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12340 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12342 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12343 VBOXSTRICTRC_VAL(rcStrict)));
12344 break;
12345 }
12346
12347 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12348 {
12349 AssertRCReturn(rc, rc);
12350 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12351 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12352 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12354 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12355 break;
12356 }
12357
12358 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12359 {
12360 AssertRCReturn(rc, rc);
12361 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12362 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12363 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12364 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12366 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12367 break;
12368 }
12369
12370 default:
12371 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12372 VERR_VMX_UNEXPECTED_EXCEPTION);
12373 }
12374
12375 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12376 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12377 NOREF(pVM);
12378 return rcStrict;
12379}
12380
12381
12382/**
12383 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12384 * VM-exit.
12385 */
12386HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12387{
12388 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12389 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12390
12391 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12392 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12393 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12394 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12395 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12396 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12397 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12398 AssertRCReturn(rc2, rc2);
12399
12400 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12401 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12402 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12403 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12404 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12405 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12406 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12407 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12408 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12409
12410 /* I/O operation lookup arrays. */
12411 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12412 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12413
12414 VBOXSTRICTRC rcStrict;
12415 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12416 uint32_t const cbInstr = pVmxTransient->cbInstr;
12417 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12418 PVM pVM = pVCpu->CTX_SUFF(pVM);
12419 if (fIOString)
12420 {
12421#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12422 See @bugref{5752#c158}. Should work now. */
12423 /*
12424 * INS/OUTS - I/O String instruction.
12425 *
12426 * Use instruction-information if available, otherwise fall back on
12427 * interpreting the instruction.
12428 */
12429 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12430 fIOWrite ? 'w' : 'r'));
12431 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12432 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12433 {
12434 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12435 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12436 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12437 AssertRCReturn(rc2, rc2);
12438 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12439 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12440 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12441 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12442 if (fIOWrite)
12443 {
12444 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12445 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12446 }
12447 else
12448 {
12449 /*
12450 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12451 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12452 * See Intel Instruction spec. for "INS".
12453 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12454 */
12455 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12456 }
12457 }
12458 else
12459 {
12460 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12461 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12462 AssertRCReturn(rc2, rc2);
12463 rcStrict = IEMExecOne(pVCpu);
12464 }
12465 /** @todo IEM needs to be setting these flags somehow. */
12466 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12467 fUpdateRipAlready = true;
12468#else
12469 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12470 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12471 if (RT_SUCCESS(rcStrict))
12472 {
12473 if (fIOWrite)
12474 {
12475 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12476 (DISCPUMODE)pDis->uAddrMode, cbValue);
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12478 }
12479 else
12480 {
12481 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12482 (DISCPUMODE)pDis->uAddrMode, cbValue);
12483 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12484 }
12485 }
12486 else
12487 {
12488 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12489 pMixedCtx->rip));
12490 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12491 }
12492#endif
12493 }
12494 else
12495 {
12496 /*
12497 * IN/OUT - I/O instruction.
12498 */
12499 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12500 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12501 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12502 if (fIOWrite)
12503 {
12504 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12505 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12506 }
12507 else
12508 {
12509 uint32_t u32Result = 0;
12510 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12511 if (IOM_SUCCESS(rcStrict))
12512 {
12513 /* Save result of I/O IN instr. in AL/AX/EAX. */
12514 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12515 }
12516 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12517 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12519 }
12520 }
12521
12522 if (IOM_SUCCESS(rcStrict))
12523 {
12524 if (!fUpdateRipAlready)
12525 {
12526 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12527 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12528 }
12529
12530 /*
12531 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12532 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12533 */
12534 if (fIOString)
12535 {
12536 /** @todo Single-step for INS/OUTS with REP prefix? */
12537 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12538 }
12539 else if ( !fDbgStepping
12540 && fGstStepping)
12541 {
12542 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12543 }
12544
12545 /*
12546 * If any I/O breakpoints are armed, we need to check if one triggered
12547 * and take appropriate action.
12548 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12549 */
12550 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12551 AssertRCReturn(rc2, rc2);
12552
12553 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12554 * execution engines about whether hyper BPs and such are pending. */
12555 uint32_t const uDr7 = pMixedCtx->dr[7];
12556 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12557 && X86_DR7_ANY_RW_IO(uDr7)
12558 && (pMixedCtx->cr4 & X86_CR4_DE))
12559 || DBGFBpIsHwIoArmed(pVM)))
12560 {
12561 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12562
12563 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12564 VMMRZCallRing3Disable(pVCpu);
12565 HM_DISABLE_PREEMPT();
12566
12567 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12568
12569 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12570 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12571 {
12572 /* Raise #DB. */
12573 if (fIsGuestDbgActive)
12574 ASMSetDR6(pMixedCtx->dr[6]);
12575 if (pMixedCtx->dr[7] != uDr7)
12576 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12577
12578 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12579 }
12580 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12581 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12582 else if ( rcStrict2 != VINF_SUCCESS
12583 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12584 rcStrict = rcStrict2;
12585 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12586
12587 HM_RESTORE_PREEMPT();
12588 VMMRZCallRing3Enable(pVCpu);
12589 }
12590 }
12591
12592#ifdef VBOX_STRICT
12593 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12594 Assert(!fIOWrite);
12595 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12596 Assert(fIOWrite);
12597 else
12598 {
12599#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12600 * statuses, that the VMM device and some others may return. See
12601 * IOM_SUCCESS() for guidance. */
12602 AssertMsg( RT_FAILURE(rcStrict)
12603 || rcStrict == VINF_SUCCESS
12604 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12605 || rcStrict == VINF_EM_DBG_BREAKPOINT
12606 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12607 || rcStrict == VINF_EM_RAW_TO_R3
12608 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12609#endif
12610 }
12611#endif
12612
12613 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12614 return rcStrict;
12615}
12616
12617
12618/**
12619 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12620 * VM-exit.
12621 */
12622HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12623{
12624 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12625
12626 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12627 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12628 AssertRCReturn(rc, rc);
12629 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12630 {
12631 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12632 AssertRCReturn(rc, rc);
12633 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12634 {
12635 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12636
12637 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12638 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12639
12640 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12641 Assert(!pVCpu->hm.s.Event.fPending);
12642 pVCpu->hm.s.Event.fPending = true;
12643 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12644 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12645 AssertRCReturn(rc, rc);
12646 if (fErrorCodeValid)
12647 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12648 else
12649 pVCpu->hm.s.Event.u32ErrCode = 0;
12650 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12651 && uVector == X86_XCPT_PF)
12652 {
12653 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12654 }
12655
12656 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12657 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12658 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12659 }
12660 }
12661
12662 /* Fall back to the interpreter to emulate the task-switch. */
12663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12664 return VERR_EM_INTERPRETER;
12665}
12666
12667
12668/**
12669 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12670 */
12671HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12672{
12673 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12674 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12675 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12676 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12677 AssertRCReturn(rc, rc);
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12679 return VINF_EM_DBG_STEPPED;
12680}
12681
12682
12683/**
12684 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12685 */
12686HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12687{
12688 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12689
12690 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12691
12692 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12693 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12694 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12695 {
12696 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12697 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12698 {
12699 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12700 return VERR_EM_INTERPRETER;
12701 }
12702 }
12703 else
12704 {
12705 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12706 rcStrict1 = VINF_SUCCESS;
12707 return rcStrict1;
12708 }
12709
12710#if 0
12711 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12712 * just sync the whole thing. */
12713 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12714#else
12715 /* Aggressive state sync. for now. */
12716 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12717 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12718 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12719#endif
12720 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12721 AssertRCReturn(rc, rc);
12722
12723 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12724 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12725 VBOXSTRICTRC rcStrict2;
12726 switch (uAccessType)
12727 {
12728 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12729 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12730 {
12731 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12732 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12733 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12734
12735 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12736 GCPhys &= PAGE_BASE_GC_MASK;
12737 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12738 PVM pVM = pVCpu->CTX_SUFF(pVM);
12739 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12740 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12741
12742 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12743 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12744 CPUMCTX2CORE(pMixedCtx), GCPhys);
12745 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12746 if ( rcStrict2 == VINF_SUCCESS
12747 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12748 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12749 {
12750 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12751 | HM_CHANGED_GUEST_RSP
12752 | HM_CHANGED_GUEST_RFLAGS
12753 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12754 rcStrict2 = VINF_SUCCESS;
12755 }
12756 break;
12757 }
12758
12759 default:
12760 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12761 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12762 break;
12763 }
12764
12765 if (rcStrict2 != VINF_SUCCESS)
12766 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12767 return rcStrict2;
12768}
12769
12770
12771/**
12772 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12773 * VM-exit.
12774 */
12775HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12776{
12777 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12778
12779 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12780 if (pVmxTransient->fWasGuestDebugStateActive)
12781 {
12782 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12783 HMVMX_RETURN_UNEXPECTED_EXIT();
12784 }
12785
12786 if ( !pVCpu->hm.s.fSingleInstruction
12787 && !pVmxTransient->fWasHyperDebugStateActive)
12788 {
12789 Assert(!DBGFIsStepping(pVCpu));
12790 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12791
12792 /* Don't intercept MOV DRx any more. */
12793 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12794 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12795 AssertRCReturn(rc, rc);
12796
12797 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12798 VMMRZCallRing3Disable(pVCpu);
12799 HM_DISABLE_PREEMPT();
12800
12801 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12802 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12803 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12804
12805 HM_RESTORE_PREEMPT();
12806 VMMRZCallRing3Enable(pVCpu);
12807
12808#ifdef VBOX_WITH_STATISTICS
12809 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12810 AssertRCReturn(rc, rc);
12811 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12813 else
12814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12815#endif
12816 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12817 return VINF_SUCCESS;
12818 }
12819
12820 /*
12821 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12822 * Update the segment registers and DR7 from the CPU.
12823 */
12824 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12825 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12826 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12827 AssertRCReturn(rc, rc);
12828 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12829
12830 PVM pVM = pVCpu->CTX_SUFF(pVM);
12831 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12832 {
12833 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12834 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12835 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12836 if (RT_SUCCESS(rc))
12837 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12838 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12839 }
12840 else
12841 {
12842 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12843 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12844 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12845 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12846 }
12847
12848 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12849 if (RT_SUCCESS(rc))
12850 {
12851 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12852 AssertRCReturn(rc2, rc2);
12853 return VINF_SUCCESS;
12854 }
12855 return rc;
12856}
12857
12858
12859/**
12860 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12861 * Conditional VM-exit.
12862 */
12863HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12864{
12865 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12866 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12867
12868 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12869 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12870 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12871 {
12872 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12873 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12874 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12875 {
12876 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12877 return VERR_EM_INTERPRETER;
12878 }
12879 }
12880 else
12881 {
12882 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12883 rcStrict1 = VINF_SUCCESS;
12884 return rcStrict1;
12885 }
12886
12887 RTGCPHYS GCPhys = 0;
12888 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12889
12890#if 0
12891 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12892#else
12893 /* Aggressive state sync. for now. */
12894 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12895 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12896 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12897#endif
12898 AssertRCReturn(rc, rc);
12899
12900 /*
12901 * If we succeed, resume guest execution.
12902 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12903 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12904 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12905 * weird case. See @bugref{6043}.
12906 */
12907 PVM pVM = pVCpu->CTX_SUFF(pVM);
12908 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12909 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12910 if ( rcStrict2 == VINF_SUCCESS
12911 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12912 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12913 {
12914 /* Successfully handled MMIO operation. */
12915 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12916 | HM_CHANGED_GUEST_RSP
12917 | HM_CHANGED_GUEST_RFLAGS
12918 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12919 return VINF_SUCCESS;
12920 }
12921 return rcStrict2;
12922}
12923
12924
12925/**
12926 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12927 * VM-exit.
12928 */
12929HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12930{
12931 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12932 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12933
12934 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12935 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12936 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12937 {
12938 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12939 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12940 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12941 }
12942 else
12943 {
12944 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12945 rcStrict1 = VINF_SUCCESS;
12946 return rcStrict1;
12947 }
12948
12949 RTGCPHYS GCPhys = 0;
12950 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12951 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12952#if 0
12953 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12954#else
12955 /* Aggressive state sync. for now. */
12956 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12957 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12958 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12959#endif
12960 AssertRCReturn(rc, rc);
12961
12962 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12963 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12964
12965 RTGCUINT uErrorCode = 0;
12966 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12967 uErrorCode |= X86_TRAP_PF_ID;
12968 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12969 uErrorCode |= X86_TRAP_PF_RW;
12970 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12971 uErrorCode |= X86_TRAP_PF_P;
12972
12973 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12974
12975 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12976 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12977
12978 /* Handle the pagefault trap for the nested shadow table. */
12979 PVM pVM = pVCpu->CTX_SUFF(pVM);
12980 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12981 TRPMResetTrap(pVCpu);
12982
12983 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12984 if ( rcStrict2 == VINF_SUCCESS
12985 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12986 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12987 {
12988 /* Successfully synced our nested page tables. */
12989 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12990 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12991 | HM_CHANGED_GUEST_RSP
12992 | HM_CHANGED_GUEST_RFLAGS);
12993 return VINF_SUCCESS;
12994 }
12995
12996 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12997 return rcStrict2;
12998}
12999
13000/** @} */
13001
13002/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13003/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13004/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13005
13006/** @name VM-exit exception handlers.
13007 * @{
13008 */
13009
13010/**
13011 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13012 */
13013static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13014{
13015 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13017
13018 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13019 AssertRCReturn(rc, rc);
13020
13021 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13022 {
13023 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13024 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13025
13026 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13027 * provides VM-exit instruction length. If this causes problem later,
13028 * disassemble the instruction like it's done on AMD-V. */
13029 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13030 AssertRCReturn(rc2, rc2);
13031 return rc;
13032 }
13033
13034 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13035 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13036 return rc;
13037}
13038
13039
13040/**
13041 * VM-exit exception handler for \#BP (Breakpoint exception).
13042 */
13043static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13044{
13045 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13047
13048 /** @todo Try optimize this by not saving the entire guest state unless
13049 * really needed. */
13050 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13051 AssertRCReturn(rc, rc);
13052
13053 PVM pVM = pVCpu->CTX_SUFF(pVM);
13054 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13055 if (rc == VINF_EM_RAW_GUEST_TRAP)
13056 {
13057 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13058 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13059 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13060 AssertRCReturn(rc, rc);
13061
13062 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13063 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13064 }
13065
13066 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13067 return rc;
13068}
13069
13070
13071/**
13072 * VM-exit exception handler for \#AC (alignment check exception).
13073 */
13074static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13075{
13076 RT_NOREF_PV(pMixedCtx);
13077 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13078
13079 /*
13080 * Re-inject it. We'll detect any nesting before getting here.
13081 */
13082 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13083 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13084 AssertRCReturn(rc, rc);
13085 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13086
13087 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13088 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13089 return VINF_SUCCESS;
13090}
13091
13092
13093/**
13094 * VM-exit exception handler for \#DB (Debug exception).
13095 */
13096static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13097{
13098 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13100 Log6(("XcptDB\n"));
13101
13102 /*
13103 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13104 * for processing.
13105 */
13106 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13107 AssertRCReturn(rc, rc);
13108
13109 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13110 uint64_t uDR6 = X86_DR6_INIT_VAL;
13111 uDR6 |= ( pVmxTransient->uExitQualification
13112 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13113
13114 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13115 if (rc == VINF_EM_RAW_GUEST_TRAP)
13116 {
13117 /*
13118 * The exception was for the guest. Update DR6, DR7.GD and
13119 * IA32_DEBUGCTL.LBR before forwarding it.
13120 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13121 */
13122 VMMRZCallRing3Disable(pVCpu);
13123 HM_DISABLE_PREEMPT();
13124
13125 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13126 pMixedCtx->dr[6] |= uDR6;
13127 if (CPUMIsGuestDebugStateActive(pVCpu))
13128 ASMSetDR6(pMixedCtx->dr[6]);
13129
13130 HM_RESTORE_PREEMPT();
13131 VMMRZCallRing3Enable(pVCpu);
13132
13133 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13134 AssertRCReturn(rc, rc);
13135
13136 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13137 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13138
13139 /* Paranoia. */
13140 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13141 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13142
13143 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13144 AssertRCReturn(rc, rc);
13145
13146 /*
13147 * Raise #DB in the guest.
13148 *
13149 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13150 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13151 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13152 *
13153 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13154 */
13155 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13156 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13157 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13158 AssertRCReturn(rc, rc);
13159 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13160 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13161 return VINF_SUCCESS;
13162 }
13163
13164 /*
13165 * Not a guest trap, must be a hypervisor related debug event then.
13166 * Update DR6 in case someone is interested in it.
13167 */
13168 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13169 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13170 CPUMSetHyperDR6(pVCpu, uDR6);
13171
13172 return rc;
13173}
13174
13175
13176/**
13177 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13178 * point exception).
13179 */
13180static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13181{
13182 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13183
13184 /* We require CR0 and EFER. EFER is always up-to-date. */
13185 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13186 AssertRCReturn(rc, rc);
13187
13188 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13189 VMMRZCallRing3Disable(pVCpu);
13190 HM_DISABLE_PREEMPT();
13191
13192 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13193 if (pVmxTransient->fWasGuestFPUStateActive)
13194 {
13195 rc = VINF_EM_RAW_GUEST_TRAP;
13196 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13197 }
13198 else
13199 {
13200#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13201 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13202#endif
13203 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13204 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13205 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13206 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13207 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13208 }
13209
13210 HM_RESTORE_PREEMPT();
13211 VMMRZCallRing3Enable(pVCpu);
13212
13213 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13214 {
13215 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13216 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13217 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13218 pVCpu->hm.s.fPreloadGuestFpu = true;
13219 }
13220 else
13221 {
13222 /* Forward #NM to the guest. */
13223 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13224 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13225 AssertRCReturn(rc, rc);
13226 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13227 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13228 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13229 }
13230
13231 return VINF_SUCCESS;
13232}
13233
13234
13235/**
13236 * VM-exit exception handler for \#GP (General-protection exception).
13237 *
13238 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13239 */
13240static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13241{
13242 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13243 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13244
13245 int rc;
13246 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13247 { /* likely */ }
13248 else
13249 {
13250#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13251 Assert(pVCpu->hm.s.fUsingDebugLoop);
13252#endif
13253 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13254 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13255 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13256 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13257 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13258 AssertRCReturn(rc, rc);
13259 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13260 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13261 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13262 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13263 return rc;
13264 }
13265
13266 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13267 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13268
13269 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13270 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13271 AssertRCReturn(rc, rc);
13272
13273 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13274 uint32_t cbOp = 0;
13275 PVM pVM = pVCpu->CTX_SUFF(pVM);
13276 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13277 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13278 if (RT_SUCCESS(rc))
13279 {
13280 rc = VINF_SUCCESS;
13281 Assert(cbOp == pDis->cbInstr);
13282 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13283 switch (pDis->pCurInstr->uOpcode)
13284 {
13285 case OP_CLI:
13286 {
13287 pMixedCtx->eflags.Bits.u1IF = 0;
13288 pMixedCtx->eflags.Bits.u1RF = 0;
13289 pMixedCtx->rip += pDis->cbInstr;
13290 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13291 if ( !fDbgStepping
13292 && pMixedCtx->eflags.Bits.u1TF)
13293 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13294 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13295 break;
13296 }
13297
13298 case OP_STI:
13299 {
13300 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13301 pMixedCtx->eflags.Bits.u1IF = 1;
13302 pMixedCtx->eflags.Bits.u1RF = 0;
13303 pMixedCtx->rip += pDis->cbInstr;
13304 if (!fOldIF)
13305 {
13306 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13307 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13308 }
13309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13310 if ( !fDbgStepping
13311 && pMixedCtx->eflags.Bits.u1TF)
13312 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13314 break;
13315 }
13316
13317 case OP_HLT:
13318 {
13319 rc = VINF_EM_HALT;
13320 pMixedCtx->rip += pDis->cbInstr;
13321 pMixedCtx->eflags.Bits.u1RF = 0;
13322 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13324 break;
13325 }
13326
13327 case OP_POPF:
13328 {
13329 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13330 uint32_t cbParm;
13331 uint32_t uMask;
13332 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13333 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13334 {
13335 cbParm = 4;
13336 uMask = 0xffffffff;
13337 }
13338 else
13339 {
13340 cbParm = 2;
13341 uMask = 0xffff;
13342 }
13343
13344 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13345 RTGCPTR GCPtrStack = 0;
13346 X86EFLAGS Eflags;
13347 Eflags.u32 = 0;
13348 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13349 &GCPtrStack);
13350 if (RT_SUCCESS(rc))
13351 {
13352 Assert(sizeof(Eflags.u32) >= cbParm);
13353 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13354 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13355 }
13356 if (RT_FAILURE(rc))
13357 {
13358 rc = VERR_EM_INTERPRETER;
13359 break;
13360 }
13361 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13362 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13363 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13364 pMixedCtx->esp += cbParm;
13365 pMixedCtx->esp &= uMask;
13366 pMixedCtx->rip += pDis->cbInstr;
13367 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13368 | HM_CHANGED_GUEST_RSP
13369 | HM_CHANGED_GUEST_RFLAGS);
13370 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13371 POPF restores EFLAGS.TF. */
13372 if ( !fDbgStepping
13373 && fGstStepping)
13374 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13376 break;
13377 }
13378
13379 case OP_PUSHF:
13380 {
13381 uint32_t cbParm;
13382 uint32_t uMask;
13383 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13384 {
13385 cbParm = 4;
13386 uMask = 0xffffffff;
13387 }
13388 else
13389 {
13390 cbParm = 2;
13391 uMask = 0xffff;
13392 }
13393
13394 /* Get the stack pointer & push the contents of eflags onto the stack. */
13395 RTGCPTR GCPtrStack = 0;
13396 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13397 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13398 if (RT_FAILURE(rc))
13399 {
13400 rc = VERR_EM_INTERPRETER;
13401 break;
13402 }
13403 X86EFLAGS Eflags = pMixedCtx->eflags;
13404 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13405 Eflags.Bits.u1RF = 0;
13406 Eflags.Bits.u1VM = 0;
13407
13408 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13409 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13410 {
13411 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13412 rc = VERR_EM_INTERPRETER;
13413 break;
13414 }
13415 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13416 pMixedCtx->esp -= cbParm;
13417 pMixedCtx->esp &= uMask;
13418 pMixedCtx->rip += pDis->cbInstr;
13419 pMixedCtx->eflags.Bits.u1RF = 0;
13420 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13421 | HM_CHANGED_GUEST_RSP
13422 | HM_CHANGED_GUEST_RFLAGS);
13423 if ( !fDbgStepping
13424 && pMixedCtx->eflags.Bits.u1TF)
13425 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13426 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13427 break;
13428 }
13429
13430 case OP_IRET:
13431 {
13432 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13433 * instruction reference. */
13434 RTGCPTR GCPtrStack = 0;
13435 uint32_t uMask = 0xffff;
13436 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13437 uint16_t aIretFrame[3];
13438 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13439 {
13440 rc = VERR_EM_INTERPRETER;
13441 break;
13442 }
13443 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13444 &GCPtrStack);
13445 if (RT_SUCCESS(rc))
13446 {
13447 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13448 PGMACCESSORIGIN_HM));
13449 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13450 }
13451 if (RT_FAILURE(rc))
13452 {
13453 rc = VERR_EM_INTERPRETER;
13454 break;
13455 }
13456 pMixedCtx->eip = 0;
13457 pMixedCtx->ip = aIretFrame[0];
13458 pMixedCtx->cs.Sel = aIretFrame[1];
13459 pMixedCtx->cs.ValidSel = aIretFrame[1];
13460 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13461 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13462 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13463 pMixedCtx->sp += sizeof(aIretFrame);
13464 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13465 | HM_CHANGED_GUEST_SEGMENT_REGS
13466 | HM_CHANGED_GUEST_RSP
13467 | HM_CHANGED_GUEST_RFLAGS);
13468 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13469 if ( !fDbgStepping
13470 && fGstStepping)
13471 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13472 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13474 break;
13475 }
13476
13477 case OP_INT:
13478 {
13479 uint16_t uVector = pDis->Param1.uValue & 0xff;
13480 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13481 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13483 break;
13484 }
13485
13486 case OP_INTO:
13487 {
13488 if (pMixedCtx->eflags.Bits.u1OF)
13489 {
13490 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13491 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13492 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13493 }
13494 else
13495 {
13496 pMixedCtx->eflags.Bits.u1RF = 0;
13497 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13498 }
13499 break;
13500 }
13501
13502 default:
13503 {
13504 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13505 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13506 EMCODETYPE_SUPERVISOR);
13507 rc = VBOXSTRICTRC_VAL(rc2);
13508 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13509 /** @todo We have to set pending-debug exceptions here when the guest is
13510 * single-stepping depending on the instruction that was interpreted. */
13511 Log4(("#GP rc=%Rrc\n", rc));
13512 break;
13513 }
13514 }
13515 }
13516 else
13517 rc = VERR_EM_INTERPRETER;
13518
13519 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13520 ("#GP Unexpected rc=%Rrc\n", rc));
13521 return rc;
13522}
13523
13524
13525/**
13526 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13527 * the exception reported in the VMX transient structure back into the VM.
13528 *
13529 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13530 * up-to-date.
13531 */
13532static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13533{
13534 RT_NOREF_PV(pMixedCtx);
13535 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13536#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13537 Assert(pVCpu->hm.s.fUsingDebugLoop);
13538#endif
13539
13540 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13541 hmR0VmxCheckExitDueToEventDelivery(). */
13542 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13543 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13544 AssertRCReturn(rc, rc);
13545 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13546
13547#ifdef DEBUG_ramshankar
13548 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13549 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13550 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13551#endif
13552
13553 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13554 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13555 return VINF_SUCCESS;
13556}
13557
13558
13559/**
13560 * VM-exit exception handler for \#PF (Page-fault exception).
13561 */
13562static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13563{
13564 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13565 PVM pVM = pVCpu->CTX_SUFF(pVM);
13566 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13567 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13568 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13569 AssertRCReturn(rc, rc);
13570
13571 if (!pVM->hm.s.fNestedPaging)
13572 { /* likely */ }
13573 else
13574 {
13575#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13576 Assert(pVCpu->hm.s.fUsingDebugLoop);
13577#endif
13578 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13579 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13580 {
13581 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13582 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13583 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13584 }
13585 else
13586 {
13587 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13588 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13589 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13590 }
13591 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13592 return rc;
13593 }
13594
13595 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13596 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13597 if (pVmxTransient->fVectoringPF)
13598 {
13599 Assert(pVCpu->hm.s.Event.fPending);
13600 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13601 }
13602
13603 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13604 AssertRCReturn(rc, rc);
13605
13606 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13607 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13608
13609 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13610 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13611 (RTGCPTR)pVmxTransient->uExitQualification);
13612
13613 Log4(("#PF: rc=%Rrc\n", rc));
13614 if (rc == VINF_SUCCESS)
13615 {
13616#if 0
13617 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13618 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13619 * memory? We don't update the whole state here... */
13620 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13621 | HM_CHANGED_GUEST_RSP
13622 | HM_CHANGED_GUEST_RFLAGS
13623 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13624#else
13625 /*
13626 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13627 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13628 */
13629 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13630 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13631#endif
13632 TRPMResetTrap(pVCpu);
13633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13634 return rc;
13635 }
13636
13637 if (rc == VINF_EM_RAW_GUEST_TRAP)
13638 {
13639 if (!pVmxTransient->fVectoringDoublePF)
13640 {
13641 /* It's a guest page fault and needs to be reflected to the guest. */
13642 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13643 TRPMResetTrap(pVCpu);
13644 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13645 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13646 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13647 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13648 }
13649 else
13650 {
13651 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13652 TRPMResetTrap(pVCpu);
13653 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13654 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13655 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13656 }
13657
13658 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13659 return VINF_SUCCESS;
13660 }
13661
13662 TRPMResetTrap(pVCpu);
13663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13664 return rc;
13665}
13666
13667/** @} */
13668
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette