VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 65222

Last change on this file since 65222 was 65222, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Fix longjump when mapping the APIC access page, map the page just once.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 584.9 KB
Line 
1/* $Id: HMVMXR0.cpp 65222 2017-01-10 12:33:45Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Saves a set of guest MSRs back into the guest-CPU context.
1524 *
1525 * @param pVCpu The cross context virtual CPU structure.
1526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1527 * out-of-sync. Make sure to update the required fields
1528 * before using them.
1529 *
1530 * @remarks No-long-jump zone!!!
1531 */
1532static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1533{
1534 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1535 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1536
1537 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1538 {
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1544 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1545 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1546 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1547 }
1548#else
1549 NOREF(pMixedCtx);
1550#endif
1551 }
1552}
1553
1554
1555/**
1556 * Loads a set of guests MSRs to allow read/passthru to the guest.
1557 *
1558 * The name of this function is slightly confusing. This function does NOT
1559 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1560 * common prefix for functions dealing with "lazy restoration" of the shared
1561 * MSRs.
1562 *
1563 * @param pVCpu The cross context virtual CPU structure.
1564 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1565 * out-of-sync. Make sure to update the required fields
1566 * before using them.
1567 *
1568 * @remarks No-long-jump zone!!!
1569 */
1570static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1571{
1572 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1574
1575#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1576 do { \
1577 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1578 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1579 else \
1580 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1581 } while (0)
1582
1583 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1584 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1585 {
1586#if HC_ARCH_BITS == 64
1587 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1588 {
1589 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1590 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1593 }
1594#else
1595 RT_NOREF(pMixedCtx);
1596#endif
1597 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1598 }
1599
1600#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1601}
1602
1603
1604/**
1605 * Performs lazy restoration of the set of host MSRs if they were previously
1606 * loaded with guest MSR values.
1607 *
1608 * @param pVCpu The cross context virtual CPU structure.
1609 *
1610 * @remarks No-long-jump zone!!!
1611 * @remarks The guest MSRs should have been saved back into the guest-CPU
1612 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1613 */
1614static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1615{
1616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1618
1619 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1620 {
1621 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1622#if HC_ARCH_BITS == 64
1623 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1624 {
1625 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1626 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1627 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1628 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1629 }
1630#endif
1631 }
1632 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1633}
1634
1635
1636/**
1637 * Verifies that our cached values of the VMCS controls are all
1638 * consistent with what's actually present in the VMCS.
1639 *
1640 * @returns VBox status code.
1641 * @param pVCpu The cross context virtual CPU structure.
1642 */
1643static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1644{
1645 uint32_t u32Val;
1646 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1649 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1650
1651 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1652 AssertRCReturn(rc, rc);
1653 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1654 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1659 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1660
1661 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1662 AssertRCReturn(rc, rc);
1663 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1664 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1665
1666 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1667 {
1668 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1669 AssertRCReturn(rc, rc);
1670 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1671 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1672 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1673 }
1674
1675 return VINF_SUCCESS;
1676}
1677
1678
1679#ifdef VBOX_STRICT
1680/**
1681 * Verifies that our cached host EFER value has not changed
1682 * since we cached it.
1683 *
1684 * @param pVCpu The cross context virtual CPU structure.
1685 */
1686static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1687{
1688 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1689
1690 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1691 {
1692 uint64_t u64Val;
1693 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1694 AssertRC(rc);
1695
1696 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1697 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1698 }
1699}
1700
1701
1702/**
1703 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1704 * VMCS are correct.
1705 *
1706 * @param pVCpu The cross context virtual CPU structure.
1707 */
1708static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1709{
1710 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1711
1712 /* Verify MSR counts in the VMCS are what we think it should be. */
1713 uint32_t cMsrs;
1714 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1718 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1719
1720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1721 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1722
1723 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1724 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1725 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1726 {
1727 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1728 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1729 pGuestMsr->u32Msr, cMsrs));
1730
1731 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1732 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1733 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1734
1735 /* Verify that the permissions are as expected in the MSR bitmap. */
1736 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1737 {
1738 VMXMSREXITREAD enmRead;
1739 VMXMSREXITWRITE enmWrite;
1740 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1741 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1742 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1743 {
1744 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1746 }
1747 else
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1750 pGuestMsr->u32Msr, cMsrs));
1751 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 }
1754 }
1755 }
1756}
1757#endif /* VBOX_STRICT */
1758
1759
1760/**
1761 * Flushes the TLB using EPT.
1762 *
1763 * @returns VBox status code.
1764 * @param pVCpu The cross context virtual CPU structure of the calling
1765 * EMT. Can be NULL depending on @a enmFlush.
1766 * @param enmFlush Type of flush.
1767 *
1768 * @remarks Caller is responsible for making sure this function is called only
1769 * when NestedPaging is supported and providing @a enmFlush that is
1770 * supported by the CPU.
1771 * @remarks Can be called with interrupts disabled.
1772 */
1773static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1774{
1775 uint64_t au64Descriptor[2];
1776 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1777 au64Descriptor[0] = 0;
1778 else
1779 {
1780 Assert(pVCpu);
1781 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1782 }
1783 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1784
1785 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1786 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1787 rc));
1788 if ( RT_SUCCESS(rc)
1789 && pVCpu)
1790 {
1791 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1792 }
1793}
1794
1795
1796/**
1797 * Flushes the TLB using VPID.
1798 *
1799 * @returns VBox status code.
1800 * @param pVM The cross context VM structure.
1801 * @param pVCpu The cross context virtual CPU structure of the calling
1802 * EMT. Can be NULL depending on @a enmFlush.
1803 * @param enmFlush Type of flush.
1804 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1805 * on @a enmFlush).
1806 *
1807 * @remarks Can be called with interrupts disabled.
1808 */
1809static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1810{
1811 NOREF(pVM);
1812 AssertPtr(pVM);
1813 Assert(pVM->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833 if ( RT_SUCCESS(rc)
1834 && pVCpu)
1835 {
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 }
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVM The cross context VM structure.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVM);
1853 AssertPtr(pVCpu);
1854 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1855
1856 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1857 if (!fFlushPending)
1858 {
1859 /*
1860 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1861 * See @bugref{6043} and @bugref{6177}.
1862 *
1863 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1864 * function maybe called in a loop with individual addresses.
1865 */
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1869 {
1870 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1871 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1872 }
1873 else
1874 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1875 }
1876 else if (pVM->hm.s.fNestedPaging)
1877 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1878 }
1879
1880 return VINF_SUCCESS;
1881}
1882
1883
1884/**
1885 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1886 * otherwise there is nothing really to invalidate.
1887 *
1888 * @returns VBox status code.
1889 * @param pVM The cross context VM structure.
1890 * @param pVCpu The cross context virtual CPU structure.
1891 * @param GCPhys Guest physical address of the page to invalidate.
1892 */
1893VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1894{
1895 NOREF(pVM); NOREF(GCPhys);
1896 LogFlowFunc(("%RGp\n", GCPhys));
1897
1898 /*
1899 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1900 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1901 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1902 */
1903 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1904 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1911 * case where neither EPT nor VPID is supported by the CPU.
1912 *
1913 * @param pVM The cross context VM structure.
1914 * @param pVCpu The cross context virtual CPU structure.
1915 * @param pCpu Pointer to the global HM struct.
1916 *
1917 * @remarks Called with interrupts disabled.
1918 */
1919static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1920{
1921 AssertPtr(pVCpu);
1922 AssertPtr(pCpu);
1923 NOREF(pVM);
1924
1925 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1926
1927 Assert(pCpu->idCpu != NIL_RTCPUID);
1928 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1929 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1930 pVCpu->hm.s.fForceTLBFlush = false;
1931 return;
1932}
1933
1934
1935/**
1936 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1937 *
1938 * @param pVM The cross context VM structure.
1939 * @param pVCpu The cross context virtual CPU structure.
1940 * @param pCpu Pointer to the global HM CPU struct.
1941 * @remarks All references to "ASID" in this function pertains to "VPID" in
1942 * Intel's nomenclature. The reason is, to avoid confusion in compare
1943 * statements since the host-CPU copies are named "ASID".
1944 *
1945 * @remarks Called with interrupts disabled.
1946 */
1947static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1948{
1949#ifdef VBOX_WITH_STATISTICS
1950 bool fTlbFlushed = false;
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1953 if (!fTlbFlushed) \
1954 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1955 } while (0)
1956#else
1957# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1958# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1959#endif
1960
1961 AssertPtr(pVM);
1962 AssertPtr(pCpu);
1963 AssertPtr(pVCpu);
1964 Assert(pCpu->idCpu != NIL_RTCPUID);
1965
1966 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1967 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1968 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1969
1970 /*
1971 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1972 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1973 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1974 */
1975 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1976 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1977 {
1978 ++pCpu->uCurrentAsid;
1979 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1980 {
1981 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1982 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1983 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1984 }
1985
1986 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1987 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1988 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1989
1990 /*
1991 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1992 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1993 */
1994 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1995 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1996 HMVMX_SET_TAGGED_TLB_FLUSHED();
1997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1998 }
1999
2000 /* Check for explicit TLB flushes. */
2001 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2002 {
2003 /*
2004 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2005 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2006 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2007 * but not guest-physical mappings.
2008 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2009 */
2010 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2011 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2012 HMVMX_SET_TAGGED_TLB_FLUSHED();
2013 }
2014
2015 pVCpu->hm.s.fForceTLBFlush = false;
2016 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2017
2018 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2019 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2020 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2021 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2022 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2023 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2024 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2025 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2026 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2027
2028 /* Update VMCS with the VPID. */
2029 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2030 AssertRC(rc);
2031
2032#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2033}
2034
2035
2036/**
2037 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2038 *
2039 * @returns VBox status code.
2040 * @param pVM The cross context VM structure.
2041 * @param pVCpu The cross context virtual CPU structure.
2042 * @param pCpu Pointer to the global HM CPU struct.
2043 *
2044 * @remarks Called with interrupts disabled.
2045 */
2046static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2047{
2048 AssertPtr(pVM);
2049 AssertPtr(pVCpu);
2050 AssertPtr(pCpu);
2051 Assert(pCpu->idCpu != NIL_RTCPUID);
2052 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2053 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2054
2055 /*
2056 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2057 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2058 */
2059 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2060 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2061 {
2062 pVCpu->hm.s.fForceTLBFlush = true;
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2064 }
2065
2066 /* Check for explicit TLB flushes. */
2067 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2068 {
2069 pVCpu->hm.s.fForceTLBFlush = true;
2070 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2071 }
2072
2073 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2074 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2075
2076 if (pVCpu->hm.s.fForceTLBFlush)
2077 {
2078 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2079 pVCpu->hm.s.fForceTLBFlush = false;
2080 }
2081}
2082
2083
2084/**
2085 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2086 *
2087 * @returns VBox status code.
2088 * @param pVM The cross context VM structure.
2089 * @param pVCpu The cross context virtual CPU structure.
2090 * @param pCpu Pointer to the global HM CPU struct.
2091 *
2092 * @remarks Called with interrupts disabled.
2093 */
2094static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2095{
2096 AssertPtr(pVM);
2097 AssertPtr(pVCpu);
2098 AssertPtr(pCpu);
2099 Assert(pCpu->idCpu != NIL_RTCPUID);
2100 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2101 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2102
2103 /*
2104 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2105 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2106 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2107 */
2108 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2109 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2110 {
2111 pVCpu->hm.s.fForceTLBFlush = true;
2112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2113 }
2114
2115 /* Check for explicit TLB flushes. */
2116 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2117 {
2118 /*
2119 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2120 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2121 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2122 */
2123 pVCpu->hm.s.fForceTLBFlush = true;
2124 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2125 }
2126
2127 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2128 if (pVCpu->hm.s.fForceTLBFlush)
2129 {
2130 ++pCpu->uCurrentAsid;
2131 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2132 {
2133 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2134 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2135 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2136 }
2137
2138 pVCpu->hm.s.fForceTLBFlush = false;
2139 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2140 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2141 if (pCpu->fFlushAsidBeforeUse)
2142 {
2143 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2144 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2145 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2146 {
2147 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2148 pCpu->fFlushAsidBeforeUse = false;
2149 }
2150 else
2151 {
2152 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2153 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2154 }
2155 }
2156 }
2157
2158 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2159 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2160 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2161 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2162 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2163 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2164 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2165
2166 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2167 AssertRC(rc);
2168}
2169
2170
2171/**
2172 * Flushes the guest TLB entry based on CPU capabilities.
2173 *
2174 * @param pVCpu The cross context virtual CPU structure.
2175 * @param pCpu Pointer to the global HM CPU struct.
2176 */
2177DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2178{
2179#ifdef HMVMX_ALWAYS_FLUSH_TLB
2180 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2181#endif
2182 PVM pVM = pVCpu->CTX_SUFF(pVM);
2183 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2184 {
2185 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2186 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2187 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2189 default:
2190 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2191 break;
2192 }
2193
2194 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2195}
2196
2197
2198/**
2199 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2200 * TLB entries from the host TLB before VM-entry.
2201 *
2202 * @returns VBox status code.
2203 * @param pVM The cross context VM structure.
2204 */
2205static int hmR0VmxSetupTaggedTlb(PVM pVM)
2206{
2207 /*
2208 * Determine optimal flush type for Nested Paging.
2209 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2210 * guest execution (see hmR3InitFinalizeR0()).
2211 */
2212 if (pVM->hm.s.fNestedPaging)
2213 {
2214 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2218 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2220 else
2221 {
2222 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2223 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227
2228 /* Make sure the write-back cacheable memory type for EPT is supported. */
2229 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2230 {
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235
2236 /* EPT requires a page-walk length of 4. */
2237 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2238 {
2239 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2240 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2241 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2242 }
2243 }
2244 else
2245 {
2246 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2247 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2248 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2249 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2250 }
2251 }
2252
2253 /*
2254 * Determine optimal flush type for VPID.
2255 */
2256 if (pVM->hm.s.vmx.fVpid)
2257 {
2258 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2261 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2262 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2264 else
2265 {
2266 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2267 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2268 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2271 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2272 pVM->hm.s.vmx.fVpid = false;
2273 }
2274 }
2275 else
2276 {
2277 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2278 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2279 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2280 pVM->hm.s.vmx.fVpid = false;
2281 }
2282 }
2283
2284 /*
2285 * Setup the handler for flushing tagged-TLBs.
2286 */
2287 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2289 else if (pVM->hm.s.fNestedPaging)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2291 else if (pVM->hm.s.vmx.fVpid)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2293 else
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2295 return VINF_SUCCESS;
2296}
2297
2298
2299/**
2300 * Sets up pin-based VM-execution controls in the VMCS.
2301 *
2302 * @returns VBox status code.
2303 * @param pVM The cross context VM structure.
2304 * @param pVCpu The cross context virtual CPU structure.
2305 */
2306static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2307{
2308 AssertPtr(pVM);
2309 AssertPtr(pVCpu);
2310
2311 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2312 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2313
2314 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2315 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2316
2317 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2319
2320 /* Enable the VMX preemption timer. */
2321 if (pVM->hm.s.vmx.fUsePreemptTimer)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2325 }
2326
2327#if 0
2328 /* Enable posted-interrupt processing. */
2329 if (pVM->hm.s.fPostedIntrs)
2330 {
2331 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2332 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2333 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2334 }
2335#endif
2336
2337 if ((val & zap) != val)
2338 {
2339 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2340 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2341 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2342 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2343 }
2344
2345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2346 AssertRCReturn(rc, rc);
2347
2348 pVCpu->hm.s.vmx.u32PinCtls = val;
2349 return rc;
2350}
2351
2352
2353/**
2354 * Sets up processor-based VM-execution controls in the VMCS.
2355 *
2356 * @returns VBox status code.
2357 * @param pVM The cross context VM structure.
2358 * @param pVCpu The cross context virtual CPU structure.
2359 */
2360static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2361{
2362 AssertPtr(pVM);
2363 AssertPtr(pVCpu);
2364
2365 int rc = VERR_INTERNAL_ERROR_5;
2366 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2367 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2368
2369 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2376
2377 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2378 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2379 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2380 {
2381 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2382 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2383 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2384 }
2385
2386 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2387 if (!pVM->hm.s.fNestedPaging)
2388 {
2389 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2390 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2391 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2392 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2393 }
2394
2395 /* Use TPR shadowing if supported by the CPU. */
2396 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2397 {
2398 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2399 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2400 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2401 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2402 AssertRCReturn(rc, rc);
2403
2404 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2405 /* CR8 writes cause a VM-exit based on TPR threshold. */
2406 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2407 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2408 }
2409 else
2410 {
2411 /*
2412 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2413 * Set this control only for 64-bit guests.
2414 */
2415 if (pVM->hm.s.fAllow64BitGuests)
2416 {
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2419 }
2420 }
2421
2422 /* Use MSR-bitmaps if supported by the CPU. */
2423 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2424 {
2425 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2426
2427 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2428 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2429 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 AssertRCReturn(rc, rc);
2431
2432 /*
2433 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2434 * automatically using dedicated fields in the VMCS.
2435 */
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441
2442#if HC_ARCH_BITS == 64
2443 /*
2444 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2445 */
2446 if (pVM->hm.s.fAllow64BitGuests)
2447 {
2448 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 }
2453#endif
2454 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2455 }
2456
2457 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2458 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2459 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2460
2461 if ((val & zap) != val)
2462 {
2463 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2464 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2465 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2466 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2467 }
2468
2469 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2470 AssertRCReturn(rc, rc);
2471
2472 pVCpu->hm.s.vmx.u32ProcCtls = val;
2473
2474 /*
2475 * Secondary processor-based VM-execution controls.
2476 */
2477 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2478 {
2479 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2480 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2481
2482 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2483 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2484
2485 if (pVM->hm.s.fNestedPaging)
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2487 else
2488 {
2489 /*
2490 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2491 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2492 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2493 */
2494 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2496 }
2497
2498 if (pVM->hm.s.vmx.fVpid)
2499 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2500
2501 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2503
2504#if 0
2505 if (pVM->hm.s.fVirtApicRegs)
2506 {
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2509
2510 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2511 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2512 }
2513#endif
2514
2515 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2516 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2517 * done dynamically. */
2518 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2519 {
2520 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2521 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2522 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2523 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2524 AssertRCReturn(rc, rc);
2525 }
2526
2527 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2528 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2529
2530 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2531 && pVM->hm.s.vmx.cPleGapTicks
2532 && pVM->hm.s.vmx.cPleWindowTicks)
2533 {
2534 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2535
2536 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2537 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2538 AssertRCReturn(rc, rc);
2539 }
2540
2541 if ((val & zap) != val)
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2544 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2550 AssertRCReturn(rc, rc);
2551
2552 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2553 }
2554 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2555 {
2556 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2557 "available\n"));
2558 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2559 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560 }
2561
2562 return VINF_SUCCESS;
2563}
2564
2565
2566/**
2567 * Sets up miscellaneous (everything other than Pin & Processor-based
2568 * VM-execution) control fields in the VMCS.
2569 *
2570 * @returns VBox status code.
2571 * @param pVM The cross context VM structure.
2572 * @param pVCpu The cross context virtual CPU structure.
2573 */
2574static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2575{
2576 NOREF(pVM);
2577 AssertPtr(pVM);
2578 AssertPtr(pVCpu);
2579
2580 int rc = VERR_GENERAL_FAILURE;
2581
2582 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2583#if 0
2584 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2585 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2586 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2587
2588 /*
2589 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2590 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2591 * We thus use the exception bitmap to control it rather than use both.
2592 */
2593 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2594 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2595
2596 /** @todo Explore possibility of using IO-bitmaps. */
2597 /* All IO & IOIO instructions cause VM-exits. */
2598 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2599 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2600
2601 /* Initialize the MSR-bitmap area. */
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2603 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2605 AssertRCReturn(rc, rc);
2606#endif
2607
2608 /* Setup MSR auto-load/store area. */
2609 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2611 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2617 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 AssertRCReturn(rc, rc);
2619
2620 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2622 AssertRCReturn(rc, rc);
2623
2624 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2625#if 0
2626 /* Setup debug controls */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2628 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2629 AssertRCReturn(rc, rc);
2630#endif
2631
2632 return rc;
2633}
2634
2635
2636/**
2637 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2638 *
2639 * We shall setup those exception intercepts that don't change during the
2640 * lifetime of the VM here. The rest are done dynamically while loading the
2641 * guest state.
2642 *
2643 * @returns VBox status code.
2644 * @param pVM The cross context VM structure.
2645 * @param pVCpu The cross context virtual CPU structure.
2646 */
2647static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2648{
2649 AssertPtr(pVM);
2650 AssertPtr(pVCpu);
2651
2652 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2653
2654 uint32_t u32XcptBitmap = 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 *
3406 * @remarks Can cause longjumps!!!
3407 */
3408DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3409{
3410 NOREF(pMixedCtx);
3411
3412 int rc = VINF_SUCCESS;
3413 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3414 {
3415 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3416 && APICIsEnabled(pVCpu))
3417 {
3418 /*
3419 * Setup TPR shadowing.
3420 */
3421 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3422 {
3423 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3424
3425 bool fPendingIntr = false;
3426 uint8_t u8Tpr = 0;
3427 uint8_t u8PendingIntr = 0;
3428 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3429 AssertRCReturn(rc, rc);
3430
3431 /*
3432 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3433 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3434 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3435 */
3436 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3437 uint32_t u32TprThreshold = 0;
3438 if (fPendingIntr)
3439 {
3440 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3441 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3442 const uint8_t u8TprPriority = u8Tpr >> 4;
3443 if (u8PendingPriority <= u8TprPriority)
3444 u32TprThreshold = u8PendingPriority;
3445 }
3446
3447 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3448 AssertRCReturn(rc, rc);
3449 }
3450
3451#ifndef IEM_VERIFICATION_MODE_FULL
3452 /*
3453 * Setup the virtualized-APIC accesses.
3454 *
3455 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
3456 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
3457 */
3458 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3459 {
3460 uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3461 if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3462 {
3463 /* We only care about the APIC base MSR address and not the other bits. */
3464 PVM pVM = pVCpu->CTX_SUFF(pVM);
3465 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3466 RTGCPHYS GCPhysApicBase;
3467 GCPhysApicBase = u64MsrApicBase;
3468 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3469
3470 /*
3471 * We only need a single HC page as the APIC-access page for all VCPUs as it's used
3472 * purely for causing VM-exits and not for data access within the actual page.
3473 *
3474 * The following check ensures we do the mapping on a per-VM basis as our APIC code
3475 * does not allow different APICs to be mapped at different addresses on different VCPUs.
3476 *
3477 * In fact, we do not support remapping of the APIC base at all, see APICSetBaseMsr()
3478 * so we just map this once per-VM.
3479 */
3480 if (ASMAtomicCmpXchgU64(&pVM->hm.s.vmx.GCPhysApicBase, GCPhysApicBase, 0 /* u64Old */))
3481 {
3482 /* Unalias any existing mapping. */
3483 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3484 AssertRCReturn(rc, rc);
3485
3486 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
3487 Log4(("HM: VCPU%u: Mapped HC APIC-access page GCPhysApicBase=%#RGp\n", pVCpu->idCpu, GCPhysApicBase));
3488 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3489 AssertRCReturn(rc, rc);
3490 }
3491
3492 /* Update the per-VCPU cache of the APIC base MSR. */
3493 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3494 }
3495 }
3496#endif /* !IEM_VERIFICATION_MODE_FULL */
3497 }
3498 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3499 }
3500
3501 return rc;
3502}
3503
3504
3505/**
3506 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3507 *
3508 * @returns Guest's interruptibility-state.
3509 * @param pVCpu The cross context virtual CPU structure.
3510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3511 * out-of-sync. Make sure to update the required fields
3512 * before using them.
3513 *
3514 * @remarks No-long-jump zone!!!
3515 */
3516DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3517{
3518 /*
3519 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3520 */
3521 uint32_t uIntrState = 0;
3522 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3523 {
3524 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3525 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3526 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3527 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3528 {
3529 if (pMixedCtx->eflags.Bits.u1IF)
3530 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3531 else
3532 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3533 }
3534 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3535 {
3536 /*
3537 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3538 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3539 */
3540 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3541 }
3542 }
3543
3544 /*
3545 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3546 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3547 * setting this would block host-NMIs and IRET will not clear the blocking.
3548 *
3549 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3550 */
3551 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3552 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3553 {
3554 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3555 }
3556
3557 return uIntrState;
3558}
3559
3560
3561/**
3562 * Loads the guest's interruptibility-state into the guest-state area in the
3563 * VMCS.
3564 *
3565 * @returns VBox status code.
3566 * @param pVCpu The cross context virtual CPU structure.
3567 * @param uIntrState The interruptibility-state to set.
3568 */
3569static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3570{
3571 NOREF(pVCpu);
3572 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3573 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3574 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3575 AssertRC(rc);
3576 return rc;
3577}
3578
3579
3580/**
3581 * Loads the exception intercepts required for guest execution in the VMCS.
3582 *
3583 * @returns VBox status code.
3584 * @param pVCpu The cross context virtual CPU structure.
3585 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3586 * out-of-sync. Make sure to update the required fields
3587 * before using them.
3588 */
3589static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3590{
3591 NOREF(pMixedCtx);
3592 int rc = VINF_SUCCESS;
3593 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3594 {
3595 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3596 if (pVCpu->hm.s.fGIMTrapXcptUD)
3597 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3598#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3599 else
3600 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3601#endif
3602
3603 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3604 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3605
3606 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3607 AssertRCReturn(rc, rc);
3608
3609 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3610 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3611 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3612 }
3613 return rc;
3614}
3615
3616
3617/**
3618 * Loads the guest's RIP into the guest-state area in the VMCS.
3619 *
3620 * @returns VBox status code.
3621 * @param pVCpu The cross context virtual CPU structure.
3622 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3623 * out-of-sync. Make sure to update the required fields
3624 * before using them.
3625 *
3626 * @remarks No-long-jump zone!!!
3627 */
3628static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3629{
3630 int rc = VINF_SUCCESS;
3631 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3632 {
3633 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3634 AssertRCReturn(rc, rc);
3635
3636 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3637 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3638 HMCPU_CF_VALUE(pVCpu)));
3639 }
3640 return rc;
3641}
3642
3643
3644/**
3645 * Loads the guest's RSP into the guest-state area in the VMCS.
3646 *
3647 * @returns VBox status code.
3648 * @param pVCpu The cross context virtual CPU structure.
3649 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3650 * out-of-sync. Make sure to update the required fields
3651 * before using them.
3652 *
3653 * @remarks No-long-jump zone!!!
3654 */
3655static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3656{
3657 int rc = VINF_SUCCESS;
3658 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3659 {
3660 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3661 AssertRCReturn(rc, rc);
3662
3663 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3664 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3665 }
3666 return rc;
3667}
3668
3669
3670/**
3671 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3672 *
3673 * @returns VBox status code.
3674 * @param pVCpu The cross context virtual CPU structure.
3675 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3676 * out-of-sync. Make sure to update the required fields
3677 * before using them.
3678 *
3679 * @remarks No-long-jump zone!!!
3680 */
3681static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3682{
3683 int rc = VINF_SUCCESS;
3684 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3685 {
3686 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3687 Let us assert it as such and use 32-bit VMWRITE. */
3688 Assert(!(pMixedCtx->rflags.u64 >> 32));
3689 X86EFLAGS Eflags = pMixedCtx->eflags;
3690 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3691 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3692 * These will never be cleared/set, unless some other part of the VMM
3693 * code is buggy - in which case we're better of finding and fixing
3694 * those bugs than hiding them. */
3695 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3696 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3697 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3698 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3699
3700 /*
3701 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3702 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3703 */
3704 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3705 {
3706 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3707 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3708 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3709 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3710 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3711 }
3712
3713 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3714 AssertRCReturn(rc, rc);
3715
3716 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3717 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3718 }
3719 return rc;
3720}
3721
3722
3723/**
3724 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3725 *
3726 * @returns VBox status code.
3727 * @param pVCpu The cross context virtual CPU structure.
3728 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3729 * out-of-sync. Make sure to update the required fields
3730 * before using them.
3731 *
3732 * @remarks No-long-jump zone!!!
3733 */
3734DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3735{
3736 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3737 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3738 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3739 AssertRCReturn(rc, rc);
3740 return rc;
3741}
3742
3743
3744/**
3745 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3746 * CR0 is partially shared with the host and we have to consider the FPU bits.
3747 *
3748 * @returns VBox status code.
3749 * @param pVCpu The cross context virtual CPU structure.
3750 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3751 * out-of-sync. Make sure to update the required fields
3752 * before using them.
3753 *
3754 * @remarks No-long-jump zone!!!
3755 */
3756static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3757{
3758 /*
3759 * Guest CR0.
3760 * Guest FPU.
3761 */
3762 int rc = VINF_SUCCESS;
3763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3764 {
3765 Assert(!(pMixedCtx->cr0 >> 32));
3766 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3767 PVM pVM = pVCpu->CTX_SUFF(pVM);
3768
3769 /* The guest's view (read access) of its CR0 is unblemished. */
3770 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3771 AssertRCReturn(rc, rc);
3772 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3773
3774 /* Setup VT-x's view of the guest CR0. */
3775 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3776 if (pVM->hm.s.fNestedPaging)
3777 {
3778 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3779 {
3780 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3781 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3782 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3783 }
3784 else
3785 {
3786 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3787 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3788 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3789 }
3790
3791 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3792 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3793 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3794
3795 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3796 AssertRCReturn(rc, rc);
3797 }
3798 else
3799 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3800
3801 /*
3802 * Guest FPU bits.
3803 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3804 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3805 */
3806 u32GuestCR0 |= X86_CR0_NE;
3807 bool fInterceptNM = false;
3808 if (CPUMIsGuestFPUStateActive(pVCpu))
3809 {
3810 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3811 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3812 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3813 }
3814 else
3815 {
3816 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3817 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3818 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3819 }
3820
3821 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3822 bool fInterceptMF = false;
3823 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3824 fInterceptMF = true;
3825
3826 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3827 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3828 {
3829 Assert(PDMVmmDevHeapIsEnabled(pVM));
3830 Assert(pVM->hm.s.vmx.pRealModeTSS);
3831 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3832 fInterceptNM = true;
3833 fInterceptMF = true;
3834 }
3835 else
3836 {
3837 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3838 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3839 }
3840 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3841
3842 if (fInterceptNM)
3843 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3844 else
3845 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3846
3847 if (fInterceptMF)
3848 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3849 else
3850 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3851
3852 /* Additional intercepts for debugging, define these yourself explicitly. */
3853#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3854 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3855 | RT_BIT(X86_XCPT_BP)
3856 | RT_BIT(X86_XCPT_DE)
3857 | RT_BIT(X86_XCPT_NM)
3858 | RT_BIT(X86_XCPT_TS)
3859 | RT_BIT(X86_XCPT_UD)
3860 | RT_BIT(X86_XCPT_NP)
3861 | RT_BIT(X86_XCPT_SS)
3862 | RT_BIT(X86_XCPT_GP)
3863 | RT_BIT(X86_XCPT_PF)
3864 | RT_BIT(X86_XCPT_MF)
3865 ;
3866#elif defined(HMVMX_ALWAYS_TRAP_PF)
3867 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3868#endif
3869
3870 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3871
3872 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3873 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3874 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3875 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3876 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3877 else
3878 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3879
3880 u32GuestCR0 |= uSetCR0;
3881 u32GuestCR0 &= uZapCR0;
3882 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3883
3884 /* Write VT-x's view of the guest CR0 into the VMCS. */
3885 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3886 AssertRCReturn(rc, rc);
3887 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3888 uZapCR0));
3889
3890 /*
3891 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3892 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3893 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3894 */
3895 uint32_t u32CR0Mask = 0;
3896 u32CR0Mask = X86_CR0_PE
3897 | X86_CR0_NE
3898 | X86_CR0_WP
3899 | X86_CR0_PG
3900 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3901 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3902 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3903
3904 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3905 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3906 * and @bugref{6944}. */
3907#if 0
3908 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3909 u32CR0Mask &= ~X86_CR0_PE;
3910#endif
3911 if (pVM->hm.s.fNestedPaging)
3912 u32CR0Mask &= ~X86_CR0_WP;
3913
3914 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3915 if (fInterceptNM)
3916 {
3917 u32CR0Mask |= X86_CR0_TS
3918 | X86_CR0_MP;
3919 }
3920
3921 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3922 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3923 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3924 AssertRCReturn(rc, rc);
3925 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3926
3927 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3928 }
3929 return rc;
3930}
3931
3932
3933/**
3934 * Loads the guest control registers (CR3, CR4) into the guest-state area
3935 * in the VMCS.
3936 *
3937 * @returns VBox strict status code.
3938 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3939 * without unrestricted guest access and the VMMDev is not presently
3940 * mapped (e.g. EFI32).
3941 *
3942 * @param pVCpu The cross context virtual CPU structure.
3943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3944 * out-of-sync. Make sure to update the required fields
3945 * before using them.
3946 *
3947 * @remarks No-long-jump zone!!!
3948 */
3949static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3950{
3951 int rc = VINF_SUCCESS;
3952 PVM pVM = pVCpu->CTX_SUFF(pVM);
3953
3954 /*
3955 * Guest CR2.
3956 * It's always loaded in the assembler code. Nothing to do here.
3957 */
3958
3959 /*
3960 * Guest CR3.
3961 */
3962 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3963 {
3964 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3965 if (pVM->hm.s.fNestedPaging)
3966 {
3967 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3968
3969 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3970 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3971 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3972 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3973
3974 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3975 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3976 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3977
3978 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3979 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3980 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3981 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3982 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3983 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3984 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3985
3986 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3987 AssertRCReturn(rc, rc);
3988 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3989
3990 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3991 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3992 {
3993 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3994 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3995 {
3996 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3997 AssertRCReturn(rc, rc);
3998 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3999 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4000 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4001 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4002 AssertRCReturn(rc, rc);
4003 }
4004
4005 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
4006 have Unrestricted Execution to handle the guest when it's not using paging. */
4007 GCPhysGuestCR3 = pMixedCtx->cr3;
4008 }
4009 else
4010 {
4011 /*
4012 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4013 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4014 * EPT takes care of translating it to host-physical addresses.
4015 */
4016 RTGCPHYS GCPhys;
4017 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4018
4019 /* We obtain it here every time as the guest could have relocated this PCI region. */
4020 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4021 if (RT_SUCCESS(rc))
4022 { /* likely */ }
4023 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4024 {
4025 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4026 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4027 }
4028 else
4029 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4030
4031 GCPhysGuestCR3 = GCPhys;
4032 }
4033
4034 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4035 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4036 }
4037 else
4038 {
4039 /* Non-nested paging case, just use the hypervisor's CR3. */
4040 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4041
4042 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4043 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4044 }
4045 AssertRCReturn(rc, rc);
4046
4047 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4048 }
4049
4050 /*
4051 * Guest CR4.
4052 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4053 */
4054 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4055 {
4056 Assert(!(pMixedCtx->cr4 >> 32));
4057 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4058
4059 /* The guest's view of its CR4 is unblemished. */
4060 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4061 AssertRCReturn(rc, rc);
4062 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4063
4064 /* Setup VT-x's view of the guest CR4. */
4065 /*
4066 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4067 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4068 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4069 */
4070 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4071 {
4072 Assert(pVM->hm.s.vmx.pRealModeTSS);
4073 Assert(PDMVmmDevHeapIsEnabled(pVM));
4074 u32GuestCR4 &= ~X86_CR4_VME;
4075 }
4076
4077 if (pVM->hm.s.fNestedPaging)
4078 {
4079 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4080 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4081 {
4082 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4083 u32GuestCR4 |= X86_CR4_PSE;
4084 /* Our identity mapping is a 32-bit page directory. */
4085 u32GuestCR4 &= ~X86_CR4_PAE;
4086 }
4087 /* else use guest CR4.*/
4088 }
4089 else
4090 {
4091 /*
4092 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4093 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4094 */
4095 switch (pVCpu->hm.s.enmShadowMode)
4096 {
4097 case PGMMODE_REAL: /* Real-mode. */
4098 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4099 case PGMMODE_32_BIT: /* 32-bit paging. */
4100 {
4101 u32GuestCR4 &= ~X86_CR4_PAE;
4102 break;
4103 }
4104
4105 case PGMMODE_PAE: /* PAE paging. */
4106 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4107 {
4108 u32GuestCR4 |= X86_CR4_PAE;
4109 break;
4110 }
4111
4112 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4113 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4114#ifdef VBOX_ENABLE_64_BITS_GUESTS
4115 break;
4116#endif
4117 default:
4118 AssertFailed();
4119 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4120 }
4121 }
4122
4123 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4124 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4125 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4126 u32GuestCR4 |= uSetCR4;
4127 u32GuestCR4 &= uZapCR4;
4128
4129 /* Write VT-x's view of the guest CR4 into the VMCS. */
4130 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4131 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4132 AssertRCReturn(rc, rc);
4133
4134 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4135 uint32_t u32CR4Mask = X86_CR4_VME
4136 | X86_CR4_PAE
4137 | X86_CR4_PGE
4138 | X86_CR4_PSE
4139 | X86_CR4_VMXE;
4140 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4141 u32CR4Mask |= X86_CR4_OSXSAVE;
4142 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4143 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4144 AssertRCReturn(rc, rc);
4145
4146 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4147 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4148
4149 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4150 }
4151 return rc;
4152}
4153
4154
4155/**
4156 * Loads the guest debug registers into the guest-state area in the VMCS.
4157 *
4158 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4159 *
4160 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4161 *
4162 * @returns VBox status code.
4163 * @param pVCpu The cross context virtual CPU structure.
4164 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4165 * out-of-sync. Make sure to update the required fields
4166 * before using them.
4167 *
4168 * @remarks No-long-jump zone!!!
4169 */
4170static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4171{
4172 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4173 return VINF_SUCCESS;
4174
4175#ifdef VBOX_STRICT
4176 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4177 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4178 {
4179 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4180 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4181 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4182 }
4183#endif
4184
4185 int rc;
4186 PVM pVM = pVCpu->CTX_SUFF(pVM);
4187 bool fSteppingDB = false;
4188 bool fInterceptMovDRx = false;
4189 if (pVCpu->hm.s.fSingleInstruction)
4190 {
4191 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4192 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4193 {
4194 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4195 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4196 AssertRCReturn(rc, rc);
4197 Assert(fSteppingDB == false);
4198 }
4199 else
4200 {
4201 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4202 pVCpu->hm.s.fClearTrapFlag = true;
4203 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4204 fSteppingDB = true;
4205 }
4206 }
4207
4208 if ( fSteppingDB
4209 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4210 {
4211 /*
4212 * Use the combined guest and host DRx values found in the hypervisor
4213 * register set because the debugger has breakpoints active or someone
4214 * is single stepping on the host side without a monitor trap flag.
4215 *
4216 * Note! DBGF expects a clean DR6 state before executing guest code.
4217 */
4218#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4219 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4220 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4221 {
4222 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4223 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4224 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4225 }
4226 else
4227#endif
4228 if (!CPUMIsHyperDebugStateActive(pVCpu))
4229 {
4230 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4231 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4232 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4233 }
4234
4235 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4236 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4237 AssertRCReturn(rc, rc);
4238
4239 pVCpu->hm.s.fUsingHyperDR7 = true;
4240 fInterceptMovDRx = true;
4241 }
4242 else
4243 {
4244 /*
4245 * If the guest has enabled debug registers, we need to load them prior to
4246 * executing guest code so they'll trigger at the right time.
4247 */
4248 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4249 {
4250#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4251 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4252 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4253 {
4254 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4255 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4256 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4257 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4258 }
4259 else
4260#endif
4261 if (!CPUMIsGuestDebugStateActive(pVCpu))
4262 {
4263 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4264 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4265 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4266 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4267 }
4268 Assert(!fInterceptMovDRx);
4269 }
4270 /*
4271 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4272 * must intercept #DB in order to maintain a correct DR6 guest value, and
4273 * because we need to intercept it to prevent nested #DBs from hanging the
4274 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4275 */
4276#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4277 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4278 && !CPUMIsGuestDebugStateActive(pVCpu))
4279#else
4280 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4281#endif
4282 {
4283 fInterceptMovDRx = true;
4284 }
4285
4286 /* Update guest DR7. */
4287 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4288 AssertRCReturn(rc, rc);
4289
4290 pVCpu->hm.s.fUsingHyperDR7 = false;
4291 }
4292
4293 /*
4294 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4295 */
4296 if (fInterceptMovDRx)
4297 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4298 else
4299 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4300 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4301 AssertRCReturn(rc, rc);
4302
4303 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4304 return VINF_SUCCESS;
4305}
4306
4307
4308#ifdef VBOX_STRICT
4309/**
4310 * Strict function to validate segment registers.
4311 *
4312 * @remarks ASSUMES CR0 is up to date.
4313 */
4314static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4315{
4316 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4317 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4318 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4319 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4320 && ( !CPUMIsGuestInRealModeEx(pCtx)
4321 && !CPUMIsGuestInV86ModeEx(pCtx)))
4322 {
4323 /* Protected mode checks */
4324 /* CS */
4325 Assert(pCtx->cs.Attr.n.u1Present);
4326 Assert(!(pCtx->cs.Attr.u & 0xf00));
4327 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4328 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4329 || !(pCtx->cs.Attr.n.u1Granularity));
4330 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4331 || (pCtx->cs.Attr.n.u1Granularity));
4332 /* CS cannot be loaded with NULL in protected mode. */
4333 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4334 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4335 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4336 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4337 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4338 else
4339 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4340 /* SS */
4341 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4342 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4343 if ( !(pCtx->cr0 & X86_CR0_PE)
4344 || pCtx->cs.Attr.n.u4Type == 3)
4345 {
4346 Assert(!pCtx->ss.Attr.n.u2Dpl);
4347 }
4348 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4349 {
4350 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4351 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4352 Assert(pCtx->ss.Attr.n.u1Present);
4353 Assert(!(pCtx->ss.Attr.u & 0xf00));
4354 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4355 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4356 || !(pCtx->ss.Attr.n.u1Granularity));
4357 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4358 || (pCtx->ss.Attr.n.u1Granularity));
4359 }
4360 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4361 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4362 {
4363 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4364 Assert(pCtx->ds.Attr.n.u1Present);
4365 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4366 Assert(!(pCtx->ds.Attr.u & 0xf00));
4367 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4368 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4369 || !(pCtx->ds.Attr.n.u1Granularity));
4370 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4371 || (pCtx->ds.Attr.n.u1Granularity));
4372 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4373 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4374 }
4375 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4376 {
4377 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4378 Assert(pCtx->es.Attr.n.u1Present);
4379 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4380 Assert(!(pCtx->es.Attr.u & 0xf00));
4381 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4382 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4383 || !(pCtx->es.Attr.n.u1Granularity));
4384 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4385 || (pCtx->es.Attr.n.u1Granularity));
4386 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4387 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4388 }
4389 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4390 {
4391 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4392 Assert(pCtx->fs.Attr.n.u1Present);
4393 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4394 Assert(!(pCtx->fs.Attr.u & 0xf00));
4395 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4396 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4397 || !(pCtx->fs.Attr.n.u1Granularity));
4398 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4399 || (pCtx->fs.Attr.n.u1Granularity));
4400 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4401 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4402 }
4403 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4404 {
4405 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4406 Assert(pCtx->gs.Attr.n.u1Present);
4407 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4408 Assert(!(pCtx->gs.Attr.u & 0xf00));
4409 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4410 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4411 || !(pCtx->gs.Attr.n.u1Granularity));
4412 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4413 || (pCtx->gs.Attr.n.u1Granularity));
4414 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4415 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4416 }
4417 /* 64-bit capable CPUs. */
4418# if HC_ARCH_BITS == 64
4419 Assert(!(pCtx->cs.u64Base >> 32));
4420 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4421 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4422 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4423# endif
4424 }
4425 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4426 || ( CPUMIsGuestInRealModeEx(pCtx)
4427 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4428 {
4429 /* Real and v86 mode checks. */
4430 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4431 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4432 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4433 {
4434 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4435 }
4436 else
4437 {
4438 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4439 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4440 }
4441
4442 /* CS */
4443 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4444 Assert(pCtx->cs.u32Limit == 0xffff);
4445 Assert(u32CSAttr == 0xf3);
4446 /* SS */
4447 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4448 Assert(pCtx->ss.u32Limit == 0xffff);
4449 Assert(u32SSAttr == 0xf3);
4450 /* DS */
4451 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4452 Assert(pCtx->ds.u32Limit == 0xffff);
4453 Assert(u32DSAttr == 0xf3);
4454 /* ES */
4455 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4456 Assert(pCtx->es.u32Limit == 0xffff);
4457 Assert(u32ESAttr == 0xf3);
4458 /* FS */
4459 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4460 Assert(pCtx->fs.u32Limit == 0xffff);
4461 Assert(u32FSAttr == 0xf3);
4462 /* GS */
4463 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4464 Assert(pCtx->gs.u32Limit == 0xffff);
4465 Assert(u32GSAttr == 0xf3);
4466 /* 64-bit capable CPUs. */
4467# if HC_ARCH_BITS == 64
4468 Assert(!(pCtx->cs.u64Base >> 32));
4469 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4470 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4471 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4472# endif
4473 }
4474}
4475#endif /* VBOX_STRICT */
4476
4477
4478/**
4479 * Writes a guest segment register into the guest-state area in the VMCS.
4480 *
4481 * @returns VBox status code.
4482 * @param pVCpu The cross context virtual CPU structure.
4483 * @param idxSel Index of the selector in the VMCS.
4484 * @param idxLimit Index of the segment limit in the VMCS.
4485 * @param idxBase Index of the segment base in the VMCS.
4486 * @param idxAccess Index of the access rights of the segment in the VMCS.
4487 * @param pSelReg Pointer to the segment selector.
4488 *
4489 * @remarks No-long-jump zone!!!
4490 */
4491static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4492 uint32_t idxAccess, PCPUMSELREG pSelReg)
4493{
4494 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4495 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4496 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4497 AssertRCReturn(rc, rc);
4498
4499 uint32_t u32Access = pSelReg->Attr.u;
4500 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4501 {
4502 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4503 u32Access = 0xf3;
4504 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4505 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4506 }
4507 else
4508 {
4509 /*
4510 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4511 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4512 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4513 * loaded in protected-mode have their attribute as 0.
4514 */
4515 if (!u32Access)
4516 u32Access = X86DESCATTR_UNUSABLE;
4517 }
4518
4519 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4520 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4521 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4522
4523 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4524 AssertRCReturn(rc, rc);
4525 return rc;
4526}
4527
4528
4529/**
4530 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4531 * into the guest-state area in the VMCS.
4532 *
4533 * @returns VBox status code.
4534 * @param pVCpu The cross context virtual CPU structure.
4535 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4536 * out-of-sync. Make sure to update the required fields
4537 * before using them.
4538 *
4539 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4540 * @remarks No-long-jump zone!!!
4541 */
4542static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4543{
4544 int rc = VERR_INTERNAL_ERROR_5;
4545 PVM pVM = pVCpu->CTX_SUFF(pVM);
4546
4547 /*
4548 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4549 */
4550 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4551 {
4552 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4553 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4554 {
4555 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4556 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4557 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4558 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4559 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4560 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4561 }
4562
4563#ifdef VBOX_WITH_REM
4564 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4565 {
4566 Assert(pVM->hm.s.vmx.pRealModeTSS);
4567 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4568 if ( pVCpu->hm.s.vmx.fWasInRealMode
4569 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4570 {
4571 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4572 in real-mode (e.g. OpenBSD 4.0) */
4573 REMFlushTBs(pVM);
4574 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4575 pVCpu->hm.s.vmx.fWasInRealMode = false;
4576 }
4577 }
4578#endif
4579 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4580 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4581 AssertRCReturn(rc, rc);
4582 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4583 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4584 AssertRCReturn(rc, rc);
4585 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4586 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4587 AssertRCReturn(rc, rc);
4588 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4589 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4590 AssertRCReturn(rc, rc);
4591 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4592 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4593 AssertRCReturn(rc, rc);
4594 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4595 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4596 AssertRCReturn(rc, rc);
4597
4598#ifdef VBOX_STRICT
4599 /* Validate. */
4600 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4601#endif
4602
4603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4604 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4605 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4606 }
4607
4608 /*
4609 * Guest TR.
4610 */
4611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4612 {
4613 /*
4614 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4615 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4616 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4617 */
4618 uint16_t u16Sel = 0;
4619 uint32_t u32Limit = 0;
4620 uint64_t u64Base = 0;
4621 uint32_t u32AccessRights = 0;
4622
4623 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4624 {
4625 u16Sel = pMixedCtx->tr.Sel;
4626 u32Limit = pMixedCtx->tr.u32Limit;
4627 u64Base = pMixedCtx->tr.u64Base;
4628 u32AccessRights = pMixedCtx->tr.Attr.u;
4629 }
4630 else
4631 {
4632 Assert(pVM->hm.s.vmx.pRealModeTSS);
4633 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4634
4635 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4636 RTGCPHYS GCPhys;
4637 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4638 AssertRCReturn(rc, rc);
4639
4640 X86DESCATTR DescAttr;
4641 DescAttr.u = 0;
4642 DescAttr.n.u1Present = 1;
4643 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4644
4645 u16Sel = 0;
4646 u32Limit = HM_VTX_TSS_SIZE;
4647 u64Base = GCPhys; /* in real-mode phys = virt. */
4648 u32AccessRights = DescAttr.u;
4649 }
4650
4651 /* Validate. */
4652 Assert(!(u16Sel & RT_BIT(2)));
4653 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4654 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4655 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4656 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4657 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4658 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4659 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4660 Assert( (u32Limit & 0xfff) == 0xfff
4661 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4662 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4663 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4664
4665 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4666 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4667 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4668 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4669 AssertRCReturn(rc, rc);
4670
4671 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4672 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4673 }
4674
4675 /*
4676 * Guest GDTR.
4677 */
4678 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4679 {
4680 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4681 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4682 AssertRCReturn(rc, rc);
4683
4684 /* Validate. */
4685 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4686
4687 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4688 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4689 }
4690
4691 /*
4692 * Guest LDTR.
4693 */
4694 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4695 {
4696 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4697 uint32_t u32Access = 0;
4698 if (!pMixedCtx->ldtr.Attr.u)
4699 u32Access = X86DESCATTR_UNUSABLE;
4700 else
4701 u32Access = pMixedCtx->ldtr.Attr.u;
4702
4703 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4704 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4705 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4706 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4707 AssertRCReturn(rc, rc);
4708
4709 /* Validate. */
4710 if (!(u32Access & X86DESCATTR_UNUSABLE))
4711 {
4712 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4713 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4714 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4715 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4716 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4717 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4718 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4719 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4720 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4721 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4722 }
4723
4724 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4725 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4726 }
4727
4728 /*
4729 * Guest IDTR.
4730 */
4731 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4732 {
4733 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4734 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4735 AssertRCReturn(rc, rc);
4736
4737 /* Validate. */
4738 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4739
4740 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4741 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4742 }
4743
4744 return VINF_SUCCESS;
4745}
4746
4747
4748/**
4749 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4750 * areas.
4751 *
4752 * These MSRs will automatically be loaded to the host CPU on every successful
4753 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4754 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4755 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4756 *
4757 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4758 *
4759 * @returns VBox status code.
4760 * @param pVCpu The cross context virtual CPU structure.
4761 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4762 * out-of-sync. Make sure to update the required fields
4763 * before using them.
4764 *
4765 * @remarks No-long-jump zone!!!
4766 */
4767static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4768{
4769 AssertPtr(pVCpu);
4770 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4771
4772 /*
4773 * MSRs that we use the auto-load/store MSR area in the VMCS.
4774 */
4775 PVM pVM = pVCpu->CTX_SUFF(pVM);
4776 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4777 {
4778 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4779#if HC_ARCH_BITS == 32
4780 if (pVM->hm.s.fAllow64BitGuests)
4781 {
4782 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4783 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4784 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4785 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4786 AssertRCReturn(rc, rc);
4787# ifdef LOG_ENABLED
4788 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4789 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4790 {
4791 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4792 pMsr->u64Value));
4793 }
4794# endif
4795 }
4796#endif
4797 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4798 }
4799
4800 /*
4801 * Guest Sysenter MSRs.
4802 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4803 * VM-exits on WRMSRs for these MSRs.
4804 */
4805 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4806 {
4807 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4808 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4809 }
4810
4811 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4812 {
4813 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4815 }
4816
4817 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4818 {
4819 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4821 }
4822
4823 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4824 {
4825 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4826 {
4827 /*
4828 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4829 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4830 */
4831 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4832 {
4833 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4834 AssertRCReturn(rc,rc);
4835 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4836 }
4837 else
4838 {
4839 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4840 NULL /* pfAddedAndUpdated */);
4841 AssertRCReturn(rc, rc);
4842
4843 /* We need to intercept reads too, see @bugref{7386#c16}. */
4844 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4845 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4846 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4847 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4848 }
4849 }
4850 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4851 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4852 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4853 }
4854
4855 return VINF_SUCCESS;
4856}
4857
4858
4859/**
4860 * Loads the guest activity state into the guest-state area in the VMCS.
4861 *
4862 * @returns VBox status code.
4863 * @param pVCpu The cross context virtual CPU structure.
4864 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4865 * out-of-sync. Make sure to update the required fields
4866 * before using them.
4867 *
4868 * @remarks No-long-jump zone!!!
4869 */
4870static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4871{
4872 NOREF(pMixedCtx);
4873 /** @todo See if we can make use of other states, e.g.
4874 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4875 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4876 {
4877 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4878 AssertRCReturn(rc, rc);
4879
4880 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4881 }
4882 return VINF_SUCCESS;
4883}
4884
4885
4886#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4887/**
4888 * Check if guest state allows safe use of 32-bit switcher again.
4889 *
4890 * Segment bases and protected mode structures must be 32-bit addressable
4891 * because the 32-bit switcher will ignore high dword when writing these VMCS
4892 * fields. See @bugref{8432} for details.
4893 *
4894 * @returns true if safe, false if must continue to use the 64-bit switcher.
4895 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4896 * out-of-sync. Make sure to update the required fields
4897 * before using them.
4898 *
4899 * @remarks No-long-jump zone!!!
4900 */
4901static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4902{
4903 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4904 return false;
4905 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4906 return false;
4907 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4908 return false;
4909 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4910 return false;
4911 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4912 return false;
4913 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4914 return false;
4915 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4916 return false;
4917 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4918 return false;
4919 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4920 return false;
4921 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4922 return false;
4923 /* All good, bases are 32-bit. */
4924 return true;
4925}
4926#endif
4927
4928
4929/**
4930 * Sets up the appropriate function to run guest code.
4931 *
4932 * @returns VBox status code.
4933 * @param pVCpu The cross context virtual CPU structure.
4934 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4935 * out-of-sync. Make sure to update the required fields
4936 * before using them.
4937 *
4938 * @remarks No-long-jump zone!!!
4939 */
4940static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4941{
4942 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4943 {
4944#ifndef VBOX_ENABLE_64_BITS_GUESTS
4945 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4946#endif
4947 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4948#if HC_ARCH_BITS == 32
4949 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4950 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4951 {
4952 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4953 {
4954 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4955 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4956 | HM_CHANGED_VMX_ENTRY_CTLS
4957 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4958 }
4959 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4960
4961 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4962 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4963 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4964 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4965 }
4966#else
4967 /* 64-bit host. */
4968 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4969#endif
4970 }
4971 else
4972 {
4973 /* Guest is not in long mode, use the 32-bit handler. */
4974#if HC_ARCH_BITS == 32
4975 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4976 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4977 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4978 {
4979 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4980 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4981 | HM_CHANGED_VMX_ENTRY_CTLS
4982 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4983 }
4984# ifdef VBOX_ENABLE_64_BITS_GUESTS
4985 /*
4986 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4987 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4988 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4989 * the much faster 32-bit switcher again.
4990 */
4991 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4992 {
4993 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4994 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4995 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4996 }
4997 else
4998 {
4999 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
5000 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
5001 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
5002 {
5003 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5004 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5005 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5006 | HM_CHANGED_VMX_ENTRY_CTLS
5007 | HM_CHANGED_VMX_EXIT_CTLS
5008 | HM_CHANGED_HOST_CONTEXT);
5009 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5010 }
5011 }
5012# else
5013 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5014# endif
5015#else
5016 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5017#endif
5018 }
5019 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5020 return VINF_SUCCESS;
5021}
5022
5023
5024/**
5025 * Wrapper for running the guest code in VT-x.
5026 *
5027 * @returns VBox status code, no informational status codes.
5028 * @param pVM The cross context VM structure.
5029 * @param pVCpu The cross context virtual CPU structure.
5030 * @param pCtx Pointer to the guest-CPU context.
5031 *
5032 * @remarks No-long-jump zone!!!
5033 */
5034DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5035{
5036 /*
5037 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5038 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5039 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5040 */
5041 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5042 /** @todo Add stats for resume vs launch. */
5043#ifdef VBOX_WITH_KERNEL_USING_XMM
5044 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5045#else
5046 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5047#endif
5048 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5049 return rc;
5050}
5051
5052
5053/**
5054 * Reports world-switch error and dumps some useful debug info.
5055 *
5056 * @param pVM The cross context VM structure.
5057 * @param pVCpu The cross context virtual CPU structure.
5058 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5059 * @param pCtx Pointer to the guest-CPU context.
5060 * @param pVmxTransient Pointer to the VMX transient structure (only
5061 * exitReason updated).
5062 */
5063static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5064{
5065 Assert(pVM);
5066 Assert(pVCpu);
5067 Assert(pCtx);
5068 Assert(pVmxTransient);
5069 HMVMX_ASSERT_PREEMPT_SAFE();
5070
5071 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5072 switch (rcVMRun)
5073 {
5074 case VERR_VMX_INVALID_VMXON_PTR:
5075 AssertFailed();
5076 break;
5077 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5078 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5079 {
5080 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5081 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5082 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5083 AssertRC(rc);
5084
5085 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5086 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5087 Cannot do it here as we may have been long preempted. */
5088
5089#ifdef VBOX_STRICT
5090 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5091 pVmxTransient->uExitReason));
5092 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5093 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5094 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5095 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5096 else
5097 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5098 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5099 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5100
5101 /* VMX control bits. */
5102 uint32_t u32Val;
5103 uint64_t u64Val;
5104 RTHCUINTREG uHCReg;
5105 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5106 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5107 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5108 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5109 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5110 {
5111 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5112 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5113 }
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5120 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5121 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5124 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5125 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5126 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5127 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5128 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5129 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5130 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5131 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5132 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5133 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5134 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5135 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5136 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5137 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5138 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5139 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5140 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5141 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5143 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5144 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5145 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5146 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5147 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5148 if (pVM->hm.s.fNestedPaging)
5149 {
5150 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5151 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5152 }
5153
5154 /* Guest bits. */
5155 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5156 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5157 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5158 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5159 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5160 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5161 if (pVM->hm.s.vmx.fVpid)
5162 {
5163 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5164 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5165 }
5166
5167 /* Host bits. */
5168 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5169 Log4(("Host CR0 %#RHr\n", uHCReg));
5170 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5171 Log4(("Host CR3 %#RHr\n", uHCReg));
5172 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5173 Log4(("Host CR4 %#RHr\n", uHCReg));
5174
5175 RTGDTR HostGdtr;
5176 PCX86DESCHC pDesc;
5177 ASMGetGDTR(&HostGdtr);
5178 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5179 Log4(("Host CS %#08x\n", u32Val));
5180 if (u32Val < HostGdtr.cbGdt)
5181 {
5182 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5183 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5184 }
5185
5186 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5187 Log4(("Host DS %#08x\n", u32Val));
5188 if (u32Val < HostGdtr.cbGdt)
5189 {
5190 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5191 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5192 }
5193
5194 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5195 Log4(("Host ES %#08x\n", u32Val));
5196 if (u32Val < HostGdtr.cbGdt)
5197 {
5198 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5199 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5200 }
5201
5202 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5203 Log4(("Host FS %#08x\n", u32Val));
5204 if (u32Val < HostGdtr.cbGdt)
5205 {
5206 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5207 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5208 }
5209
5210 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5211 Log4(("Host GS %#08x\n", u32Val));
5212 if (u32Val < HostGdtr.cbGdt)
5213 {
5214 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5215 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5216 }
5217
5218 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5219 Log4(("Host SS %#08x\n", u32Val));
5220 if (u32Val < HostGdtr.cbGdt)
5221 {
5222 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5223 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5224 }
5225
5226 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5227 Log4(("Host TR %#08x\n", u32Val));
5228 if (u32Val < HostGdtr.cbGdt)
5229 {
5230 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5231 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5232 }
5233
5234 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5235 Log4(("Host TR Base %#RHv\n", uHCReg));
5236 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5237 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5238 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5239 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5240 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5241 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5242 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5243 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5244 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5245 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5246 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5247 Log4(("Host RSP %#RHv\n", uHCReg));
5248 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5249 Log4(("Host RIP %#RHv\n", uHCReg));
5250# if HC_ARCH_BITS == 64
5251 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5252 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5253 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5254 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5255 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5256 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5257# endif
5258#endif /* VBOX_STRICT */
5259 break;
5260 }
5261
5262 default:
5263 /* Impossible */
5264 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5265 break;
5266 }
5267 NOREF(pVM); NOREF(pCtx);
5268}
5269
5270
5271#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5272#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5273# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5274#endif
5275#ifdef VBOX_STRICT
5276static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5277{
5278 switch (idxField)
5279 {
5280 case VMX_VMCS_GUEST_RIP:
5281 case VMX_VMCS_GUEST_RSP:
5282 case VMX_VMCS_GUEST_SYSENTER_EIP:
5283 case VMX_VMCS_GUEST_SYSENTER_ESP:
5284 case VMX_VMCS_GUEST_GDTR_BASE:
5285 case VMX_VMCS_GUEST_IDTR_BASE:
5286 case VMX_VMCS_GUEST_CS_BASE:
5287 case VMX_VMCS_GUEST_DS_BASE:
5288 case VMX_VMCS_GUEST_ES_BASE:
5289 case VMX_VMCS_GUEST_FS_BASE:
5290 case VMX_VMCS_GUEST_GS_BASE:
5291 case VMX_VMCS_GUEST_SS_BASE:
5292 case VMX_VMCS_GUEST_LDTR_BASE:
5293 case VMX_VMCS_GUEST_TR_BASE:
5294 case VMX_VMCS_GUEST_CR3:
5295 return true;
5296 }
5297 return false;
5298}
5299
5300static bool hmR0VmxIsValidReadField(uint32_t idxField)
5301{
5302 switch (idxField)
5303 {
5304 /* Read-only fields. */
5305 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5306 return true;
5307 }
5308 /* Remaining readable fields should also be writable. */
5309 return hmR0VmxIsValidWriteField(idxField);
5310}
5311#endif /* VBOX_STRICT */
5312
5313
5314/**
5315 * Executes the specified handler in 64-bit mode.
5316 *
5317 * @returns VBox status code (no informational status codes).
5318 * @param pVM The cross context VM structure.
5319 * @param pVCpu The cross context virtual CPU structure.
5320 * @param pCtx Pointer to the guest CPU context.
5321 * @param enmOp The operation to perform.
5322 * @param cParams Number of parameters.
5323 * @param paParam Array of 32-bit parameters.
5324 */
5325VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5326 uint32_t cParams, uint32_t *paParam)
5327{
5328 NOREF(pCtx);
5329
5330 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5331 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5332 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5333 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5334
5335#ifdef VBOX_STRICT
5336 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5337 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5338
5339 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5340 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5341#endif
5342
5343 /* Disable interrupts. */
5344 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5345
5346#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5347 RTCPUID idHostCpu = RTMpCpuId();
5348 CPUMR0SetLApic(pVCpu, idHostCpu);
5349#endif
5350
5351 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5352 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5353
5354 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5355 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5356 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5357
5358 /* Leave VMX Root Mode. */
5359 VMXDisable();
5360
5361 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5362
5363 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5364 CPUMSetHyperEIP(pVCpu, enmOp);
5365 for (int i = (int)cParams - 1; i >= 0; i--)
5366 CPUMPushHyper(pVCpu, paParam[i]);
5367
5368 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5369
5370 /* Call the switcher. */
5371 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5372 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5373
5374 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5375 /* Make sure the VMX instructions don't cause #UD faults. */
5376 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5377
5378 /* Re-enter VMX Root Mode */
5379 int rc2 = VMXEnable(HCPhysCpuPage);
5380 if (RT_FAILURE(rc2))
5381 {
5382 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5383 ASMSetFlags(fOldEFlags);
5384 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5385 return rc2;
5386 }
5387
5388 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5389 AssertRC(rc2);
5390 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5391 Assert(!(ASMGetFlags() & X86_EFL_IF));
5392 ASMSetFlags(fOldEFlags);
5393 return rc;
5394}
5395
5396
5397/**
5398 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5399 * supporting 64-bit guests.
5400 *
5401 * @returns VBox status code.
5402 * @param fResume Whether to VMLAUNCH or VMRESUME.
5403 * @param pCtx Pointer to the guest-CPU context.
5404 * @param pCache Pointer to the VMCS cache.
5405 * @param pVM The cross context VM structure.
5406 * @param pVCpu The cross context virtual CPU structure.
5407 */
5408DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5409{
5410 NOREF(fResume);
5411
5412 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5413 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5414
5415#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5416 pCache->uPos = 1;
5417 pCache->interPD = PGMGetInterPaeCR3(pVM);
5418 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5419#endif
5420
5421#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5422 pCache->TestIn.HCPhysCpuPage = 0;
5423 pCache->TestIn.HCPhysVmcs = 0;
5424 pCache->TestIn.pCache = 0;
5425 pCache->TestOut.HCPhysVmcs = 0;
5426 pCache->TestOut.pCache = 0;
5427 pCache->TestOut.pCtx = 0;
5428 pCache->TestOut.eflags = 0;
5429#else
5430 NOREF(pCache);
5431#endif
5432
5433 uint32_t aParam[10];
5434 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5435 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5436 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5437 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5438 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5439 aParam[5] = 0;
5440 aParam[6] = VM_RC_ADDR(pVM, pVM);
5441 aParam[7] = 0;
5442 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5443 aParam[9] = 0;
5444
5445#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5446 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5447 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5448#endif
5449 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5450
5451#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5452 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5453 Assert(pCtx->dr[4] == 10);
5454 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5455#endif
5456
5457#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5458 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5459 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5460 pVCpu->hm.s.vmx.HCPhysVmcs));
5461 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5462 pCache->TestOut.HCPhysVmcs));
5463 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5464 pCache->TestOut.pCache));
5465 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5466 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5467 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5468 pCache->TestOut.pCtx));
5469 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5470#endif
5471 return rc;
5472}
5473
5474
5475/**
5476 * Initialize the VMCS-Read cache.
5477 *
5478 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5479 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5480 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5481 * (those that have a 32-bit FULL & HIGH part).
5482 *
5483 * @returns VBox status code.
5484 * @param pVM The cross context VM structure.
5485 * @param pVCpu The cross context virtual CPU structure.
5486 */
5487static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5488{
5489#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5490{ \
5491 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5492 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5493 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5494 ++cReadFields; \
5495}
5496
5497 AssertPtr(pVM);
5498 AssertPtr(pVCpu);
5499 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5500 uint32_t cReadFields = 0;
5501
5502 /*
5503 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5504 * and serve to indicate exceptions to the rules.
5505 */
5506
5507 /* Guest-natural selector base fields. */
5508#if 0
5509 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5512#endif
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5522 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5523 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5525#if 0
5526 /* Unused natural width guest-state fields. */
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5529#endif
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5532
5533 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5534#if 0
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5536 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5540 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5541 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5542 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5543 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5544#endif
5545
5546 /* Natural width guest-state fields. */
5547 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5548#if 0
5549 /* Currently unused field. */
5550 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5551#endif
5552
5553 if (pVM->hm.s.fNestedPaging)
5554 {
5555 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5556 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5557 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5558 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5559 }
5560 else
5561 {
5562 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5563 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5564 }
5565
5566#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5567 return VINF_SUCCESS;
5568}
5569
5570
5571/**
5572 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5573 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5574 * darwin, running 64-bit guests).
5575 *
5576 * @returns VBox status code.
5577 * @param pVCpu The cross context virtual CPU structure.
5578 * @param idxField The VMCS field encoding.
5579 * @param u64Val 16, 32 or 64-bit value.
5580 */
5581VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5582{
5583 int rc;
5584 switch (idxField)
5585 {
5586 /*
5587 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5588 */
5589 /* 64-bit Control fields. */
5590 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5591 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5592 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5593 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5594 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5595 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5596 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5597 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5598 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5599 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5600 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5601 case VMX_VMCS64_CTRL_EPTP_FULL:
5602 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5603 /* 64-bit Guest-state fields. */
5604 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5605 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5606 case VMX_VMCS64_GUEST_PAT_FULL:
5607 case VMX_VMCS64_GUEST_EFER_FULL:
5608 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5609 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5610 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5611 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5612 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5613 /* 64-bit Host-state fields. */
5614 case VMX_VMCS64_HOST_PAT_FULL:
5615 case VMX_VMCS64_HOST_EFER_FULL:
5616 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5617 {
5618 rc = VMXWriteVmcs32(idxField, u64Val);
5619 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5620 break;
5621 }
5622
5623 /*
5624 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5625 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5626 */
5627 /* Natural-width Guest-state fields. */
5628 case VMX_VMCS_GUEST_CR3:
5629 case VMX_VMCS_GUEST_ES_BASE:
5630 case VMX_VMCS_GUEST_CS_BASE:
5631 case VMX_VMCS_GUEST_SS_BASE:
5632 case VMX_VMCS_GUEST_DS_BASE:
5633 case VMX_VMCS_GUEST_FS_BASE:
5634 case VMX_VMCS_GUEST_GS_BASE:
5635 case VMX_VMCS_GUEST_LDTR_BASE:
5636 case VMX_VMCS_GUEST_TR_BASE:
5637 case VMX_VMCS_GUEST_GDTR_BASE:
5638 case VMX_VMCS_GUEST_IDTR_BASE:
5639 case VMX_VMCS_GUEST_RSP:
5640 case VMX_VMCS_GUEST_RIP:
5641 case VMX_VMCS_GUEST_SYSENTER_ESP:
5642 case VMX_VMCS_GUEST_SYSENTER_EIP:
5643 {
5644 if (!(u64Val >> 32))
5645 {
5646 /* If this field is 64-bit, VT-x will zero out the top bits. */
5647 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5648 }
5649 else
5650 {
5651 /* Assert that only the 32->64 switcher case should ever come here. */
5652 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5653 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5654 }
5655 break;
5656 }
5657
5658 default:
5659 {
5660 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5661 rc = VERR_INVALID_PARAMETER;
5662 break;
5663 }
5664 }
5665 AssertRCReturn(rc, rc);
5666 return rc;
5667}
5668
5669
5670/**
5671 * Queue up a VMWRITE by using the VMCS write cache.
5672 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5673 *
5674 * @param pVCpu The cross context virtual CPU structure.
5675 * @param idxField The VMCS field encoding.
5676 * @param u64Val 16, 32 or 64-bit value.
5677 */
5678VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5679{
5680 AssertPtr(pVCpu);
5681 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5682
5683 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5684 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5685
5686 /* Make sure there are no duplicates. */
5687 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5688 {
5689 if (pCache->Write.aField[i] == idxField)
5690 {
5691 pCache->Write.aFieldVal[i] = u64Val;
5692 return VINF_SUCCESS;
5693 }
5694 }
5695
5696 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5697 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5698 pCache->Write.cValidEntries++;
5699 return VINF_SUCCESS;
5700}
5701#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5702
5703
5704/**
5705 * Sets up the usage of TSC-offsetting and updates the VMCS.
5706 *
5707 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5708 * VMX preemption timer.
5709 *
5710 * @returns VBox status code.
5711 * @param pVM The cross context VM structure.
5712 * @param pVCpu The cross context virtual CPU structure.
5713 *
5714 * @remarks No-long-jump zone!!!
5715 */
5716static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5717{
5718 int rc;
5719 bool fOffsettedTsc;
5720 bool fParavirtTsc;
5721 if (pVM->hm.s.vmx.fUsePreemptTimer)
5722 {
5723 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5724 &fOffsettedTsc, &fParavirtTsc);
5725
5726 /* Make sure the returned values have sane upper and lower boundaries. */
5727 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5728 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5729 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5730 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5731
5732 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5733 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5734 }
5735 else
5736 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5737
5738 /** @todo later optimize this to be done elsewhere and not before every
5739 * VM-entry. */
5740 if (fParavirtTsc)
5741 {
5742 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5743 information before every VM-entry, hence disable it for performance sake. */
5744#if 0
5745 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5746 AssertRC(rc);
5747#endif
5748 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5749 }
5750
5751 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5752 {
5753 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5754 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5755
5756 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5757 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5758 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5759 }
5760 else
5761 {
5762 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5763 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5764 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5765 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5766 }
5767}
5768
5769
5770/**
5771 * Determines if an exception is a contributory exception.
5772 *
5773 * Contributory exceptions are ones which can cause double-faults unless the
5774 * original exception was a benign exception. Page-fault is intentionally not
5775 * included here as it's a conditional contributory exception.
5776 *
5777 * @returns true if the exception is contributory, false otherwise.
5778 * @param uVector The exception vector.
5779 */
5780DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5781{
5782 switch (uVector)
5783 {
5784 case X86_XCPT_GP:
5785 case X86_XCPT_SS:
5786 case X86_XCPT_NP:
5787 case X86_XCPT_TS:
5788 case X86_XCPT_DE:
5789 return true;
5790 default:
5791 break;
5792 }
5793 return false;
5794}
5795
5796
5797/**
5798 * Sets an event as a pending event to be injected into the guest.
5799 *
5800 * @param pVCpu The cross context virtual CPU structure.
5801 * @param u32IntInfo The VM-entry interruption-information field.
5802 * @param cbInstr The VM-entry instruction length in bytes (for software
5803 * interrupts, exceptions and privileged software
5804 * exceptions).
5805 * @param u32ErrCode The VM-entry exception error code.
5806 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5807 * page-fault.
5808 *
5809 * @remarks Statistics counter assumes this is a guest event being injected or
5810 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5811 * always incremented.
5812 */
5813DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5814 RTGCUINTPTR GCPtrFaultAddress)
5815{
5816 Assert(!pVCpu->hm.s.Event.fPending);
5817 pVCpu->hm.s.Event.fPending = true;
5818 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5819 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5820 pVCpu->hm.s.Event.cbInstr = cbInstr;
5821 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5822}
5823
5824
5825/**
5826 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5827 *
5828 * @param pVCpu The cross context virtual CPU structure.
5829 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5830 * out-of-sync. Make sure to update the required fields
5831 * before using them.
5832 */
5833DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5834{
5835 NOREF(pMixedCtx);
5836 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5837 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5838 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5839 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5840}
5841
5842
5843/**
5844 * Handle a condition that occurred while delivering an event through the guest
5845 * IDT.
5846 *
5847 * @returns Strict VBox status code (i.e. informational status codes too).
5848 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5849 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5850 * to continue execution of the guest which will delivery the \#DF.
5851 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5852 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5853 *
5854 * @param pVCpu The cross context virtual CPU structure.
5855 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5856 * out-of-sync. Make sure to update the required fields
5857 * before using them.
5858 * @param pVmxTransient Pointer to the VMX transient structure.
5859 *
5860 * @remarks No-long-jump zone!!!
5861 */
5862static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5863{
5864 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5865
5866 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5867 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5868
5869 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5870 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5871 {
5872 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5873 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5874
5875 typedef enum
5876 {
5877 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5878 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5879 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5880 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5881 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5882 } VMXREFLECTXCPT;
5883
5884 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5885 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5886 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5887 {
5888 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5889 {
5890 enmReflect = VMXREFLECTXCPT_XCPT;
5891#ifdef VBOX_STRICT
5892 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5893 && uExitVector == X86_XCPT_PF)
5894 {
5895 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5896 }
5897#endif
5898 if ( uExitVector == X86_XCPT_PF
5899 && uIdtVector == X86_XCPT_PF)
5900 {
5901 pVmxTransient->fVectoringDoublePF = true;
5902 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5903 }
5904 else if ( uExitVector == X86_XCPT_AC
5905 && uIdtVector == X86_XCPT_AC)
5906 {
5907 enmReflect = VMXREFLECTXCPT_HANG;
5908 Log4(("IDT: Nested #AC - Bad guest\n"));
5909 }
5910 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5911 && hmR0VmxIsContributoryXcpt(uExitVector)
5912 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5913 || uIdtVector == X86_XCPT_PF))
5914 {
5915 enmReflect = VMXREFLECTXCPT_DF;
5916 }
5917 else if (uIdtVector == X86_XCPT_DF)
5918 enmReflect = VMXREFLECTXCPT_TF;
5919 }
5920 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5921 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5922 {
5923 /*
5924 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5925 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5926 */
5927 enmReflect = VMXREFLECTXCPT_XCPT;
5928
5929 if (uExitVector == X86_XCPT_PF)
5930 {
5931 pVmxTransient->fVectoringPF = true;
5932 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5933 }
5934 }
5935 }
5936 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5937 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5938 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5939 {
5940 /*
5941 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5942 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5943 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5944 */
5945 enmReflect = VMXREFLECTXCPT_XCPT;
5946 }
5947
5948 /*
5949 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5950 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5951 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5952 *
5953 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5954 */
5955 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5956 && enmReflect == VMXREFLECTXCPT_XCPT
5957 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5958 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5959 {
5960 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5961 }
5962
5963 switch (enmReflect)
5964 {
5965 case VMXREFLECTXCPT_XCPT:
5966 {
5967 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5968 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5969 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5970
5971 uint32_t u32ErrCode = 0;
5972 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5973 {
5974 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5975 AssertRCReturn(rc2, rc2);
5976 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5977 }
5978
5979 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5980 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5981 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5982 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5983 rcStrict = VINF_SUCCESS;
5984 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5985 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5986
5987 break;
5988 }
5989
5990 case VMXREFLECTXCPT_DF:
5991 {
5992 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5993 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5994 rcStrict = VINF_HM_DOUBLE_FAULT;
5995 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5996 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5997
5998 break;
5999 }
6000
6001 case VMXREFLECTXCPT_TF:
6002 {
6003 rcStrict = VINF_EM_RESET;
6004 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6005 uExitVector));
6006 break;
6007 }
6008
6009 case VMXREFLECTXCPT_HANG:
6010 {
6011 rcStrict = VERR_EM_GUEST_CPU_HANG;
6012 break;
6013 }
6014
6015 default:
6016 Assert(rcStrict == VINF_SUCCESS);
6017 break;
6018 }
6019 }
6020 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6021 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6022 && uExitVector != X86_XCPT_DF
6023 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6024 {
6025 /*
6026 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6027 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6028 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6029 */
6030 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6031 {
6032 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6033 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6034 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6035 }
6036 }
6037
6038 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6039 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6040 return rcStrict;
6041}
6042
6043
6044/**
6045 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6046 *
6047 * @returns VBox status code.
6048 * @param pVCpu The cross context virtual CPU structure.
6049 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6050 * out-of-sync. Make sure to update the required fields
6051 * before using them.
6052 *
6053 * @remarks No-long-jump zone!!!
6054 */
6055static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6056{
6057 NOREF(pMixedCtx);
6058
6059 /*
6060 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6061 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6062 */
6063 VMMRZCallRing3Disable(pVCpu);
6064 HM_DISABLE_PREEMPT();
6065
6066 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6067 {
6068 uint32_t uVal = 0;
6069 uint32_t uShadow = 0;
6070 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6071 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6072 AssertRCReturn(rc, rc);
6073
6074 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6075 CPUMSetGuestCR0(pVCpu, uVal);
6076 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6077 }
6078
6079 HM_RESTORE_PREEMPT();
6080 VMMRZCallRing3Enable(pVCpu);
6081 return VINF_SUCCESS;
6082}
6083
6084
6085/**
6086 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6087 *
6088 * @returns VBox status code.
6089 * @param pVCpu The cross context virtual CPU structure.
6090 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6091 * out-of-sync. Make sure to update the required fields
6092 * before using them.
6093 *
6094 * @remarks No-long-jump zone!!!
6095 */
6096static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6097{
6098 NOREF(pMixedCtx);
6099
6100 int rc = VINF_SUCCESS;
6101 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6102 {
6103 uint32_t uVal = 0;
6104 uint32_t uShadow = 0;
6105 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6106 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6107 AssertRCReturn(rc, rc);
6108
6109 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6110 CPUMSetGuestCR4(pVCpu, uVal);
6111 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6112 }
6113 return rc;
6114}
6115
6116
6117/**
6118 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6119 *
6120 * @returns VBox status code.
6121 * @param pVCpu The cross context virtual CPU structure.
6122 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6123 * out-of-sync. Make sure to update the required fields
6124 * before using them.
6125 *
6126 * @remarks No-long-jump zone!!!
6127 */
6128static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6129{
6130 int rc = VINF_SUCCESS;
6131 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6132 {
6133 uint64_t u64Val = 0;
6134 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6135 AssertRCReturn(rc, rc);
6136
6137 pMixedCtx->rip = u64Val;
6138 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6139 }
6140 return rc;
6141}
6142
6143
6144/**
6145 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6146 *
6147 * @returns VBox status code.
6148 * @param pVCpu The cross context virtual CPU structure.
6149 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6150 * out-of-sync. Make sure to update the required fields
6151 * before using them.
6152 *
6153 * @remarks No-long-jump zone!!!
6154 */
6155static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6156{
6157 int rc = VINF_SUCCESS;
6158 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6159 {
6160 uint64_t u64Val = 0;
6161 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6162 AssertRCReturn(rc, rc);
6163
6164 pMixedCtx->rsp = u64Val;
6165 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6166 }
6167 return rc;
6168}
6169
6170
6171/**
6172 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6173 *
6174 * @returns VBox status code.
6175 * @param pVCpu The cross context virtual CPU structure.
6176 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6177 * out-of-sync. Make sure to update the required fields
6178 * before using them.
6179 *
6180 * @remarks No-long-jump zone!!!
6181 */
6182static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6183{
6184 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6185 {
6186 uint32_t uVal = 0;
6187 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6188 AssertRCReturn(rc, rc);
6189
6190 pMixedCtx->eflags.u32 = uVal;
6191 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6192 {
6193 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6194 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6195
6196 pMixedCtx->eflags.Bits.u1VM = 0;
6197 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6198 }
6199
6200 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6201 }
6202 return VINF_SUCCESS;
6203}
6204
6205
6206/**
6207 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6208 * guest-CPU context.
6209 */
6210DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6211{
6212 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6213 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6214 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6215 return rc;
6216}
6217
6218
6219/**
6220 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6221 * from the guest-state area in the VMCS.
6222 *
6223 * @param pVCpu The cross context virtual CPU structure.
6224 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6225 * out-of-sync. Make sure to update the required fields
6226 * before using them.
6227 *
6228 * @remarks No-long-jump zone!!!
6229 */
6230static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6231{
6232 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6233 {
6234 uint32_t uIntrState = 0;
6235 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6236 AssertRC(rc);
6237
6238 if (!uIntrState)
6239 {
6240 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6241 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6242
6243 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6244 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6245 }
6246 else
6247 {
6248 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6249 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6250 {
6251 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6252 AssertRC(rc);
6253 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6254 AssertRC(rc);
6255
6256 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6257 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6258 }
6259 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6260 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6261
6262 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6263 {
6264 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6265 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6266 }
6267 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6268 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6269 }
6270
6271 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6272 }
6273}
6274
6275
6276/**
6277 * Saves the guest's activity state.
6278 *
6279 * @returns VBox status code.
6280 * @param pVCpu The cross context virtual CPU structure.
6281 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6282 * out-of-sync. Make sure to update the required fields
6283 * before using them.
6284 *
6285 * @remarks No-long-jump zone!!!
6286 */
6287static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6288{
6289 NOREF(pMixedCtx);
6290 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6291 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6292 return VINF_SUCCESS;
6293}
6294
6295
6296/**
6297 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6298 * the current VMCS into the guest-CPU context.
6299 *
6300 * @returns VBox status code.
6301 * @param pVCpu The cross context virtual CPU structure.
6302 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6303 * out-of-sync. Make sure to update the required fields
6304 * before using them.
6305 *
6306 * @remarks No-long-jump zone!!!
6307 */
6308static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6309{
6310 int rc = VINF_SUCCESS;
6311 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6312 {
6313 uint32_t u32Val = 0;
6314 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6315 pMixedCtx->SysEnter.cs = u32Val;
6316 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6317 }
6318
6319 uint64_t u64Val = 0;
6320 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6321 {
6322 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6323 pMixedCtx->SysEnter.eip = u64Val;
6324 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6325 }
6326 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6327 {
6328 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6329 pMixedCtx->SysEnter.esp = u64Val;
6330 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6331 }
6332 return rc;
6333}
6334
6335
6336/**
6337 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6338 * the CPU back into the guest-CPU context.
6339 *
6340 * @returns VBox status code.
6341 * @param pVCpu The cross context virtual CPU structure.
6342 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6343 * out-of-sync. Make sure to update the required fields
6344 * before using them.
6345 *
6346 * @remarks No-long-jump zone!!!
6347 */
6348static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6349{
6350 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6351 VMMRZCallRing3Disable(pVCpu);
6352 HM_DISABLE_PREEMPT();
6353
6354 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6355 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6356 {
6357 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6358 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6359 }
6360
6361 HM_RESTORE_PREEMPT();
6362 VMMRZCallRing3Enable(pVCpu);
6363
6364 return VINF_SUCCESS;
6365}
6366
6367
6368/**
6369 * Saves the auto load/store'd guest MSRs from the current VMCS into
6370 * the guest-CPU context.
6371 *
6372 * @returns VBox status code.
6373 * @param pVCpu The cross context virtual CPU structure.
6374 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6375 * out-of-sync. Make sure to update the required fields
6376 * before using them.
6377 *
6378 * @remarks No-long-jump zone!!!
6379 */
6380static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6381{
6382 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6383 return VINF_SUCCESS;
6384
6385 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6386 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6387 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6388 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6389 {
6390 switch (pMsr->u32Msr)
6391 {
6392 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6393 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6394 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6395 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6396 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6397 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6398 break;
6399
6400 default:
6401 {
6402 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6403 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6404 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6405 }
6406 }
6407 }
6408
6409 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6410 return VINF_SUCCESS;
6411}
6412
6413
6414/**
6415 * Saves the guest control registers from the current VMCS into the guest-CPU
6416 * context.
6417 *
6418 * @returns VBox status code.
6419 * @param pVCpu The cross context virtual CPU structure.
6420 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6421 * out-of-sync. Make sure to update the required fields
6422 * before using them.
6423 *
6424 * @remarks No-long-jump zone!!!
6425 */
6426static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6427{
6428 /* Guest CR0. Guest FPU. */
6429 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6430 AssertRCReturn(rc, rc);
6431
6432 /* Guest CR4. */
6433 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6434 AssertRCReturn(rc, rc);
6435
6436 /* Guest CR2 - updated always during the world-switch or in #PF. */
6437 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6438 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6439 {
6440 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6441 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6442
6443 PVM pVM = pVCpu->CTX_SUFF(pVM);
6444 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6445 || ( pVM->hm.s.fNestedPaging
6446 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6447 {
6448 uint64_t u64Val = 0;
6449 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6450 if (pMixedCtx->cr3 != u64Val)
6451 {
6452 CPUMSetGuestCR3(pVCpu, u64Val);
6453 if (VMMRZCallRing3IsEnabled(pVCpu))
6454 {
6455 PGMUpdateCR3(pVCpu, u64Val);
6456 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6457 }
6458 else
6459 {
6460 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6461 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6462 }
6463 }
6464
6465 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6466 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6467 {
6468 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6469 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6470 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6471 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6472 AssertRCReturn(rc, rc);
6473
6474 if (VMMRZCallRing3IsEnabled(pVCpu))
6475 {
6476 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6477 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6478 }
6479 else
6480 {
6481 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6482 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6483 }
6484 }
6485 }
6486
6487 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6488 }
6489
6490 /*
6491 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6492 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6493 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6494 *
6495 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6496 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6497 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6498 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6499 *
6500 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6501 */
6502 if (VMMRZCallRing3IsEnabled(pVCpu))
6503 {
6504 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6505 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6506
6507 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6508 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6509
6510 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6511 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6512 }
6513
6514 return rc;
6515}
6516
6517
6518/**
6519 * Reads a guest segment register from the current VMCS into the guest-CPU
6520 * context.
6521 *
6522 * @returns VBox status code.
6523 * @param pVCpu The cross context virtual CPU structure.
6524 * @param idxSel Index of the selector in the VMCS.
6525 * @param idxLimit Index of the segment limit in the VMCS.
6526 * @param idxBase Index of the segment base in the VMCS.
6527 * @param idxAccess Index of the access rights of the segment in the VMCS.
6528 * @param pSelReg Pointer to the segment selector.
6529 *
6530 * @remarks No-long-jump zone!!!
6531 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6532 * macro as that takes care of whether to read from the VMCS cache or
6533 * not.
6534 */
6535DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6536 PCPUMSELREG pSelReg)
6537{
6538 NOREF(pVCpu);
6539
6540 uint32_t u32Val = 0;
6541 int rc = VMXReadVmcs32(idxSel, &u32Val);
6542 AssertRCReturn(rc, rc);
6543 pSelReg->Sel = (uint16_t)u32Val;
6544 pSelReg->ValidSel = (uint16_t)u32Val;
6545 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6546
6547 rc = VMXReadVmcs32(idxLimit, &u32Val);
6548 AssertRCReturn(rc, rc);
6549 pSelReg->u32Limit = u32Val;
6550
6551 uint64_t u64Val = 0;
6552 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6553 AssertRCReturn(rc, rc);
6554 pSelReg->u64Base = u64Val;
6555
6556 rc = VMXReadVmcs32(idxAccess, &u32Val);
6557 AssertRCReturn(rc, rc);
6558 pSelReg->Attr.u = u32Val;
6559
6560 /*
6561 * If VT-x marks the segment as unusable, most other bits remain undefined:
6562 * - For CS the L, D and G bits have meaning.
6563 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6564 * - For the remaining data segments no bits are defined.
6565 *
6566 * The present bit and the unusable bit has been observed to be set at the
6567 * same time (the selector was supposed to be invalid as we started executing
6568 * a V8086 interrupt in ring-0).
6569 *
6570 * What should be important for the rest of the VBox code, is that the P bit is
6571 * cleared. Some of the other VBox code recognizes the unusable bit, but
6572 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6573 * safe side here, we'll strip off P and other bits we don't care about. If
6574 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6575 *
6576 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6577 */
6578 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6579 {
6580 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6581
6582 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6583 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6584 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6585
6586 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6587#ifdef DEBUG_bird
6588 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6589 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6590 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6591#endif
6592 }
6593 return VINF_SUCCESS;
6594}
6595
6596
6597#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6598# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6599 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6600 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6601#else
6602# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6603 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6604 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6605#endif
6606
6607
6608/**
6609 * Saves the guest segment registers from the current VMCS into the guest-CPU
6610 * context.
6611 *
6612 * @returns VBox status code.
6613 * @param pVCpu The cross context virtual CPU structure.
6614 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6615 * out-of-sync. Make sure to update the required fields
6616 * before using them.
6617 *
6618 * @remarks No-long-jump zone!!!
6619 */
6620static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6621{
6622 /* Guest segment registers. */
6623 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6624 {
6625 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6626 AssertRCReturn(rc, rc);
6627
6628 rc = VMXLOCAL_READ_SEG(CS, cs);
6629 rc |= VMXLOCAL_READ_SEG(SS, ss);
6630 rc |= VMXLOCAL_READ_SEG(DS, ds);
6631 rc |= VMXLOCAL_READ_SEG(ES, es);
6632 rc |= VMXLOCAL_READ_SEG(FS, fs);
6633 rc |= VMXLOCAL_READ_SEG(GS, gs);
6634 AssertRCReturn(rc, rc);
6635
6636 /* Restore segment attributes for real-on-v86 mode hack. */
6637 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6638 {
6639 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6640 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6641 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6642 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6643 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6644 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6645 }
6646 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6647 }
6648
6649 return VINF_SUCCESS;
6650}
6651
6652
6653/**
6654 * Saves the guest descriptor table registers and task register from the current
6655 * VMCS into the guest-CPU context.
6656 *
6657 * @returns VBox status code.
6658 * @param pVCpu The cross context virtual CPU structure.
6659 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6660 * out-of-sync. Make sure to update the required fields
6661 * before using them.
6662 *
6663 * @remarks No-long-jump zone!!!
6664 */
6665static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6666{
6667 int rc = VINF_SUCCESS;
6668
6669 /* Guest LDTR. */
6670 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6671 {
6672 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6673 AssertRCReturn(rc, rc);
6674 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6675 }
6676
6677 /* Guest GDTR. */
6678 uint64_t u64Val = 0;
6679 uint32_t u32Val = 0;
6680 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6681 {
6682 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6683 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6684 pMixedCtx->gdtr.pGdt = u64Val;
6685 pMixedCtx->gdtr.cbGdt = u32Val;
6686 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6687 }
6688
6689 /* Guest IDTR. */
6690 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6691 {
6692 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6693 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6694 pMixedCtx->idtr.pIdt = u64Val;
6695 pMixedCtx->idtr.cbIdt = u32Val;
6696 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6697 }
6698
6699 /* Guest TR. */
6700 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6701 {
6702 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6703 AssertRCReturn(rc, rc);
6704
6705 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6706 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6707 {
6708 rc = VMXLOCAL_READ_SEG(TR, tr);
6709 AssertRCReturn(rc, rc);
6710 }
6711 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6712 }
6713 return rc;
6714}
6715
6716#undef VMXLOCAL_READ_SEG
6717
6718
6719/**
6720 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6721 * context.
6722 *
6723 * @returns VBox status code.
6724 * @param pVCpu The cross context virtual CPU structure.
6725 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6726 * out-of-sync. Make sure to update the required fields
6727 * before using them.
6728 *
6729 * @remarks No-long-jump zone!!!
6730 */
6731static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6732{
6733 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6734 {
6735 if (!pVCpu->hm.s.fUsingHyperDR7)
6736 {
6737 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6738 uint32_t u32Val;
6739 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6740 pMixedCtx->dr[7] = u32Val;
6741 }
6742
6743 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6744 }
6745 return VINF_SUCCESS;
6746}
6747
6748
6749/**
6750 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6751 *
6752 * @returns VBox status code.
6753 * @param pVCpu The cross context virtual CPU structure.
6754 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6755 * out-of-sync. Make sure to update the required fields
6756 * before using them.
6757 *
6758 * @remarks No-long-jump zone!!!
6759 */
6760static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6761{
6762 NOREF(pMixedCtx);
6763
6764 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6765 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6766 return VINF_SUCCESS;
6767}
6768
6769
6770/**
6771 * Saves the entire guest state from the currently active VMCS into the
6772 * guest-CPU context.
6773 *
6774 * This essentially VMREADs all guest-data.
6775 *
6776 * @returns VBox status code.
6777 * @param pVCpu The cross context virtual CPU structure.
6778 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6779 * out-of-sync. Make sure to update the required fields
6780 * before using them.
6781 */
6782static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6783{
6784 Assert(pVCpu);
6785 Assert(pMixedCtx);
6786
6787 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6788 return VINF_SUCCESS;
6789
6790 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6791 again on the ring-3 callback path, there is no real need to. */
6792 if (VMMRZCallRing3IsEnabled(pVCpu))
6793 VMMR0LogFlushDisable(pVCpu);
6794 else
6795 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6796 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6797
6798 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6799 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6800
6801 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6802 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6803
6804 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6805 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6806
6807 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6808 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6809
6810 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6811 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6812
6813 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6814 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6815
6816 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6817 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6818
6819 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6820 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6821
6822 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6823 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6824
6825 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6826 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6827
6828 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6829 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6830 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6831
6832 if (VMMRZCallRing3IsEnabled(pVCpu))
6833 VMMR0LogFlushEnable(pVCpu);
6834
6835 return VINF_SUCCESS;
6836}
6837
6838
6839/**
6840 * Saves basic guest registers needed for IEM instruction execution.
6841 *
6842 * @returns VBox status code (OR-able).
6843 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6844 * @param pMixedCtx Pointer to the CPU context of the guest.
6845 * @param fMemory Whether the instruction being executed operates on
6846 * memory or not. Only CR0 is synced up if clear.
6847 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6848 */
6849static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6850{
6851 /*
6852 * We assume all general purpose registers other than RSP are available.
6853 *
6854 * - RIP is a must, as it will be incremented or otherwise changed.
6855 * - RFLAGS are always required to figure the CPL.
6856 * - RSP isn't always required, however it's a GPR, so frequently required.
6857 * - SS and CS are the only segment register needed if IEM doesn't do memory
6858 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6859 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6860 * be required for memory accesses.
6861 *
6862 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6863 */
6864 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6865 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6866 if (fNeedRsp)
6867 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6868 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6869 if (!fMemory)
6870 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6871 else
6872 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6873 AssertRCReturn(rc, rc);
6874 return rc;
6875}
6876
6877
6878/**
6879 * Ensures that we've got a complete basic guest-context.
6880 *
6881 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6882 * is for the interpreter.
6883 *
6884 * @returns VBox status code.
6885 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6886 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6887 * needing to be synced in.
6888 * @thread EMT(pVCpu)
6889 */
6890VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6891{
6892 /* Note! Since this is only applicable to VT-x, the implementation is placed
6893 in the VT-x part of the sources instead of the generic stuff. */
6894 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6895 {
6896 /* For now, imply that the caller might change everything too. */
6897 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6898 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6899 }
6900 return VINF_SUCCESS;
6901}
6902
6903
6904/**
6905 * Check per-VM and per-VCPU force flag actions that require us to go back to
6906 * ring-3 for one reason or another.
6907 *
6908 * @returns Strict VBox status code (i.e. informational status codes too)
6909 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6910 * ring-3.
6911 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6912 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6913 * interrupts)
6914 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6915 * all EMTs to be in ring-3.
6916 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6917 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6918 * to the EM loop.
6919 *
6920 * @param pVM The cross context VM structure.
6921 * @param pVCpu The cross context virtual CPU structure.
6922 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6923 * out-of-sync. Make sure to update the required fields
6924 * before using them.
6925 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6926 */
6927static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6928{
6929 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6930
6931 /*
6932 * Anything pending? Should be more likely than not if we're doing a good job.
6933 */
6934 if ( !fStepping
6935 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6936 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6937 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6938 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6939 return VINF_SUCCESS;
6940
6941 /* We need the control registers now, make sure the guest-CPU context is updated. */
6942 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6943 AssertRCReturn(rc3, rc3);
6944
6945 /* Pending HM CR3 sync. */
6946 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6947 {
6948 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6949 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6950 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6951 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6952 }
6953
6954 /* Pending HM PAE PDPEs. */
6955 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6956 {
6957 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6958 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6959 }
6960
6961 /* Pending PGM C3 sync. */
6962 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6963 {
6964 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6965 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6966 if (rcStrict2 != VINF_SUCCESS)
6967 {
6968 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6969 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6970 return rcStrict2;
6971 }
6972 }
6973
6974 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6975 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6976 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6977 {
6978 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6979 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6980 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6981 return rc2;
6982 }
6983
6984 /* Pending VM request packets, such as hardware interrupts. */
6985 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6986 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6987 {
6988 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6989 return VINF_EM_PENDING_REQUEST;
6990 }
6991
6992 /* Pending PGM pool flushes. */
6993 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6994 {
6995 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6996 return VINF_PGM_POOL_FLUSH_PENDING;
6997 }
6998
6999 /* Pending DMA requests. */
7000 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7001 {
7002 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7003 return VINF_EM_RAW_TO_R3;
7004 }
7005
7006 return VINF_SUCCESS;
7007}
7008
7009
7010/**
7011 * Converts any TRPM trap into a pending HM event. This is typically used when
7012 * entering from ring-3 (not longjmp returns).
7013 *
7014 * @param pVCpu The cross context virtual CPU structure.
7015 */
7016static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7017{
7018 Assert(TRPMHasTrap(pVCpu));
7019 Assert(!pVCpu->hm.s.Event.fPending);
7020
7021 uint8_t uVector;
7022 TRPMEVENT enmTrpmEvent;
7023 RTGCUINT uErrCode;
7024 RTGCUINTPTR GCPtrFaultAddress;
7025 uint8_t cbInstr;
7026
7027 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7028 AssertRC(rc);
7029
7030 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7031 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7032 if (enmTrpmEvent == TRPM_TRAP)
7033 {
7034 switch (uVector)
7035 {
7036 case X86_XCPT_NMI:
7037 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7038 break;
7039
7040 case X86_XCPT_BP:
7041 case X86_XCPT_OF:
7042 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7043 break;
7044
7045 case X86_XCPT_PF:
7046 case X86_XCPT_DF:
7047 case X86_XCPT_TS:
7048 case X86_XCPT_NP:
7049 case X86_XCPT_SS:
7050 case X86_XCPT_GP:
7051 case X86_XCPT_AC:
7052 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7053 /* no break! */
7054 default:
7055 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7056 break;
7057 }
7058 }
7059 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7060 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7061 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7062 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7063 else
7064 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7065
7066 rc = TRPMResetTrap(pVCpu);
7067 AssertRC(rc);
7068 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7069 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7070
7071 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7072}
7073
7074
7075/**
7076 * Converts the pending HM event into a TRPM trap.
7077 *
7078 * @param pVCpu The cross context virtual CPU structure.
7079 */
7080static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7081{
7082 Assert(pVCpu->hm.s.Event.fPending);
7083
7084 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7085 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7086 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7087 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7088
7089 /* If a trap was already pending, we did something wrong! */
7090 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7091
7092 TRPMEVENT enmTrapType;
7093 switch (uVectorType)
7094 {
7095 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7096 enmTrapType = TRPM_HARDWARE_INT;
7097 break;
7098
7099 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7100 enmTrapType = TRPM_SOFTWARE_INT;
7101 break;
7102
7103 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7104 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7105 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7106 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7107 enmTrapType = TRPM_TRAP;
7108 break;
7109
7110 default:
7111 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7112 enmTrapType = TRPM_32BIT_HACK;
7113 break;
7114 }
7115
7116 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7117
7118 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7119 AssertRC(rc);
7120
7121 if (fErrorCodeValid)
7122 TRPMSetErrorCode(pVCpu, uErrorCode);
7123
7124 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7125 && uVector == X86_XCPT_PF)
7126 {
7127 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7128 }
7129 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7130 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7131 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7132 {
7133 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7134 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7135 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7136 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7137 }
7138
7139 /* Clear any pending events from the VMCS. */
7140 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7141 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7142
7143 /* We're now done converting the pending event. */
7144 pVCpu->hm.s.Event.fPending = false;
7145}
7146
7147
7148/**
7149 * Does the necessary state syncing before returning to ring-3 for any reason
7150 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7151 *
7152 * @returns VBox status code.
7153 * @param pVCpu The cross context virtual CPU structure.
7154 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7155 * be out-of-sync. Make sure to update the required
7156 * fields before using them.
7157 * @param fSaveGuestState Whether to save the guest state or not.
7158 *
7159 * @remarks No-long-jmp zone!!!
7160 */
7161static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7162{
7163 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7164 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7165
7166 RTCPUID idCpu = RTMpCpuId();
7167 Log4Func(("HostCpuId=%u\n", idCpu));
7168
7169 /*
7170 * !!! IMPORTANT !!!
7171 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7172 */
7173
7174 /* Save the guest state if necessary. */
7175 if ( fSaveGuestState
7176 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7177 {
7178 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7179 AssertRCReturn(rc, rc);
7180 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7181 }
7182
7183 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7184 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7185 {
7186 if (fSaveGuestState)
7187 {
7188 /* We shouldn't reload CR0 without saving it first. */
7189 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7190 AssertRCReturn(rc, rc);
7191 }
7192 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7193 }
7194
7195 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7196#ifdef VBOX_STRICT
7197 if (CPUMIsHyperDebugStateActive(pVCpu))
7198 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7199#endif
7200 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7201 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7202 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7203 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7204
7205#if HC_ARCH_BITS == 64
7206 /* Restore host-state bits that VT-x only restores partially. */
7207 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7208 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7209 {
7210 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7211 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7212 }
7213 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7214#endif
7215
7216 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7217 if (pVCpu->hm.s.vmx.fLazyMsrs)
7218 {
7219 /* We shouldn't reload the guest MSRs without saving it first. */
7220 if (!fSaveGuestState)
7221 {
7222 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7223 AssertRCReturn(rc, rc);
7224 }
7225 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7226 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7227 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7228 }
7229
7230 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7231 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7232
7233 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7234 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7235 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7236 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7237 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7238 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7239 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7240 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7241
7242 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7243
7244 /** @todo This partially defeats the purpose of having preemption hooks.
7245 * The problem is, deregistering the hooks should be moved to a place that
7246 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7247 * context.
7248 */
7249 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7250 {
7251 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7252 AssertRCReturn(rc, rc);
7253
7254 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7255 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7256 }
7257 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7258 NOREF(idCpu);
7259
7260 return VINF_SUCCESS;
7261}
7262
7263
7264/**
7265 * Leaves the VT-x session.
7266 *
7267 * @returns VBox status code.
7268 * @param pVCpu The cross context virtual CPU structure.
7269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7270 * out-of-sync. Make sure to update the required fields
7271 * before using them.
7272 *
7273 * @remarks No-long-jmp zone!!!
7274 */
7275DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7276{
7277 HM_DISABLE_PREEMPT();
7278 HMVMX_ASSERT_CPU_SAFE();
7279 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7280 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7281
7282 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7283 and done this from the VMXR0ThreadCtxCallback(). */
7284 if (!pVCpu->hm.s.fLeaveDone)
7285 {
7286 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7287 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7288 pVCpu->hm.s.fLeaveDone = true;
7289 }
7290 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7291
7292 /*
7293 * !!! IMPORTANT !!!
7294 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7295 */
7296
7297 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7298 /** @todo Deregistering here means we need to VMCLEAR always
7299 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7300 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7301 VMMR0ThreadCtxHookDisable(pVCpu);
7302
7303 /* Leave HM context. This takes care of local init (term). */
7304 int rc = HMR0LeaveCpu(pVCpu);
7305
7306 HM_RESTORE_PREEMPT();
7307 return rc;
7308}
7309
7310
7311/**
7312 * Does the necessary state syncing before doing a longjmp to ring-3.
7313 *
7314 * @returns VBox status code.
7315 * @param pVCpu The cross context virtual CPU structure.
7316 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7317 * out-of-sync. Make sure to update the required fields
7318 * before using them.
7319 *
7320 * @remarks No-long-jmp zone!!!
7321 */
7322DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7323{
7324 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7325}
7326
7327
7328/**
7329 * Take necessary actions before going back to ring-3.
7330 *
7331 * An action requires us to go back to ring-3. This function does the necessary
7332 * steps before we can safely return to ring-3. This is not the same as longjmps
7333 * to ring-3, this is voluntary and prepares the guest so it may continue
7334 * executing outside HM (recompiler/IEM).
7335 *
7336 * @returns VBox status code.
7337 * @param pVM The cross context VM structure.
7338 * @param pVCpu The cross context virtual CPU structure.
7339 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7340 * out-of-sync. Make sure to update the required fields
7341 * before using them.
7342 * @param rcExit The reason for exiting to ring-3. Can be
7343 * VINF_VMM_UNKNOWN_RING3_CALL.
7344 */
7345static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7346{
7347 Assert(pVM);
7348 Assert(pVCpu);
7349 Assert(pMixedCtx);
7350 HMVMX_ASSERT_PREEMPT_SAFE();
7351
7352 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7353 {
7354 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7355 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7356 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7357 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7358 }
7359
7360 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7361 VMMRZCallRing3Disable(pVCpu);
7362 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7363
7364 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7365 if (pVCpu->hm.s.Event.fPending)
7366 {
7367 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7368 Assert(!pVCpu->hm.s.Event.fPending);
7369 }
7370
7371 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7372 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7373
7374 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7375 and if we're injecting an event we should have a TRPM trap pending. */
7376 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7377#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7378 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7379#endif
7380
7381 /* Save guest state and restore host state bits. */
7382 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7383 AssertRCReturn(rc, rc);
7384 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7385 /* Thread-context hooks are unregistered at this point!!! */
7386
7387 /* Sync recompiler state. */
7388 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7389 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7390 | CPUM_CHANGED_LDTR
7391 | CPUM_CHANGED_GDTR
7392 | CPUM_CHANGED_IDTR
7393 | CPUM_CHANGED_TR
7394 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7395 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7396 if ( pVM->hm.s.fNestedPaging
7397 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7398 {
7399 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7400 }
7401
7402 Assert(!pVCpu->hm.s.fClearTrapFlag);
7403
7404 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7405 if (rcExit != VINF_EM_RAW_INTERRUPT)
7406 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7407
7408 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7409
7410 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7411 VMMRZCallRing3RemoveNotification(pVCpu);
7412 VMMRZCallRing3Enable(pVCpu);
7413
7414 return rc;
7415}
7416
7417
7418/**
7419 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7420 * longjump to ring-3 and possibly get preempted.
7421 *
7422 * @returns VBox status code.
7423 * @param pVCpu The cross context virtual CPU structure.
7424 * @param enmOperation The operation causing the ring-3 longjump.
7425 * @param pvUser Opaque pointer to the guest-CPU context. The data
7426 * may be out-of-sync. Make sure to update the required
7427 * fields before using them.
7428 */
7429static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7430{
7431 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7432 {
7433 /*
7434 * !!! IMPORTANT !!!
7435 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7436 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7437 */
7438 VMMRZCallRing3RemoveNotification(pVCpu);
7439 VMMRZCallRing3Disable(pVCpu);
7440 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7441 RTThreadPreemptDisable(&PreemptState);
7442
7443 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7444 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7445
7446#if HC_ARCH_BITS == 64
7447 /* Restore host-state bits that VT-x only restores partially. */
7448 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7449 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7450 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7451 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7452#endif
7453 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7454 if (pVCpu->hm.s.vmx.fLazyMsrs)
7455 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7456
7457 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7458 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7459 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7460 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7461 {
7462 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7463 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7464 }
7465
7466 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7467 VMMR0ThreadCtxHookDisable(pVCpu);
7468 HMR0LeaveCpu(pVCpu);
7469 RTThreadPreemptRestore(&PreemptState);
7470 return VINF_SUCCESS;
7471 }
7472
7473 Assert(pVCpu);
7474 Assert(pvUser);
7475 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7476 HMVMX_ASSERT_PREEMPT_SAFE();
7477
7478 VMMRZCallRing3Disable(pVCpu);
7479 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7480
7481 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7482 enmOperation));
7483
7484 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7485 AssertRCReturn(rc, rc);
7486
7487 VMMRZCallRing3Enable(pVCpu);
7488 return VINF_SUCCESS;
7489}
7490
7491
7492/**
7493 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7494 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7495 *
7496 * @param pVCpu The cross context virtual CPU structure.
7497 */
7498DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7499{
7500 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7501 {
7502 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7503 {
7504 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7505 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7506 AssertRC(rc);
7507 Log4(("Setup interrupt-window exiting\n"));
7508 }
7509 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7510}
7511
7512
7513/**
7514 * Clears the interrupt-window exiting control in the VMCS.
7515 *
7516 * @param pVCpu The cross context virtual CPU structure.
7517 */
7518DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7519{
7520 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7521 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7522 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7523 AssertRC(rc);
7524 Log4(("Cleared interrupt-window exiting\n"));
7525}
7526
7527
7528/**
7529 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7530 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7531 *
7532 * @param pVCpu The cross context virtual CPU structure.
7533 */
7534DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7535{
7536 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7537 {
7538 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7539 {
7540 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7541 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7542 AssertRC(rc);
7543 Log4(("Setup NMI-window exiting\n"));
7544 }
7545 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7546}
7547
7548
7549/**
7550 * Clears the NMI-window exiting control in the VMCS.
7551 *
7552 * @param pVCpu The cross context virtual CPU structure.
7553 */
7554DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7555{
7556 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7557 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7558 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7559 AssertRC(rc);
7560 Log4(("Cleared NMI-window exiting\n"));
7561}
7562
7563
7564/**
7565 * Evaluates the event to be delivered to the guest and sets it as the pending
7566 * event.
7567 *
7568 * @returns The VT-x guest-interruptibility state.
7569 * @param pVCpu The cross context virtual CPU structure.
7570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7571 * out-of-sync. Make sure to update the required fields
7572 * before using them.
7573 */
7574static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7575{
7576 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7577 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7578 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7579 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7580 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7581
7582 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7583 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7584 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7585 Assert(!TRPMHasTrap(pVCpu));
7586
7587 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7588 APICUpdatePendingInterrupts(pVCpu);
7589
7590 /*
7591 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7592 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7593 */
7594 /** @todo SMI. SMIs take priority over NMIs. */
7595 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7596 {
7597 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7598 if ( !pVCpu->hm.s.Event.fPending
7599 && !fBlockNmi
7600 && !fBlockSti
7601 && !fBlockMovSS)
7602 {
7603 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7604 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7605 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7606
7607 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7608 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7609 }
7610 else
7611 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7612 }
7613 /*
7614 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7615 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7616 */
7617 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7618 && !pVCpu->hm.s.fSingleInstruction)
7619 {
7620 Assert(!DBGFIsStepping(pVCpu));
7621 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7622 AssertRC(rc);
7623 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7624 if ( !pVCpu->hm.s.Event.fPending
7625 && !fBlockInt
7626 && !fBlockSti
7627 && !fBlockMovSS)
7628 {
7629 uint8_t u8Interrupt;
7630 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7631 if (RT_SUCCESS(rc))
7632 {
7633 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7634 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7635 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7636
7637 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7638 }
7639 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7640 {
7641 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7642 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7643 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7644
7645 /*
7646 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7647 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7648 * need to re-set this force-flag here.
7649 */
7650 }
7651 else
7652 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7653 }
7654 else
7655 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7656 }
7657
7658 return uIntrState;
7659}
7660
7661
7662/**
7663 * Sets a pending-debug exception to be delivered to the guest if the guest is
7664 * single-stepping in the VMCS.
7665 *
7666 * @param pVCpu The cross context virtual CPU structure.
7667 */
7668DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7669{
7670 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7671 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7672 AssertRC(rc);
7673}
7674
7675
7676/**
7677 * Injects any pending events into the guest if the guest is in a state to
7678 * receive them.
7679 *
7680 * @returns Strict VBox status code (i.e. informational status codes too).
7681 * @param pVCpu The cross context virtual CPU structure.
7682 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7683 * out-of-sync. Make sure to update the required fields
7684 * before using them.
7685 * @param uIntrState The VT-x guest-interruptibility state.
7686 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7687 * return VINF_EM_DBG_STEPPED if the event was
7688 * dispatched directly.
7689 */
7690static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7691{
7692 HMVMX_ASSERT_PREEMPT_SAFE();
7693 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7694
7695 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7696 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7697
7698 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7699 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7700 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7701 Assert(!TRPMHasTrap(pVCpu));
7702
7703 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7704 if (pVCpu->hm.s.Event.fPending)
7705 {
7706 /*
7707 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7708 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7709 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7710 *
7711 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7712 */
7713 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7714#ifdef VBOX_STRICT
7715 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7716 {
7717 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7718 Assert(!fBlockInt);
7719 Assert(!fBlockSti);
7720 Assert(!fBlockMovSS);
7721 }
7722 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7723 {
7724 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7725 Assert(!fBlockSti);
7726 Assert(!fBlockMovSS);
7727 Assert(!fBlockNmi);
7728 }
7729#endif
7730 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7731 (uint8_t)uIntType));
7732 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7733 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7734 fStepping, &uIntrState);
7735 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7736
7737 /* Update the interruptibility-state as it could have been changed by
7738 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7739 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7740 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7741
7742 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7743 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7744 else
7745 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7746 }
7747
7748 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7749 if ( fBlockSti
7750 || fBlockMovSS)
7751 {
7752 if (!pVCpu->hm.s.fSingleInstruction)
7753 {
7754 /*
7755 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7756 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7757 * See Intel spec. 27.3.4 "Saving Non-Register State".
7758 */
7759 Assert(!DBGFIsStepping(pVCpu));
7760 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7761 AssertRCReturn(rc2, rc2);
7762 if (pMixedCtx->eflags.Bits.u1TF)
7763 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7764 }
7765 else if (pMixedCtx->eflags.Bits.u1TF)
7766 {
7767 /*
7768 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7769 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7770 */
7771 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7772 uIntrState = 0;
7773 }
7774 }
7775
7776 /*
7777 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7778 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7779 */
7780 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7781 AssertRC(rc2);
7782
7783 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7784 NOREF(fBlockMovSS); NOREF(fBlockSti);
7785 return rcStrict;
7786}
7787
7788
7789/**
7790 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7791 *
7792 * @param pVCpu The cross context virtual CPU structure.
7793 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7794 * out-of-sync. Make sure to update the required fields
7795 * before using them.
7796 */
7797DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7798{
7799 NOREF(pMixedCtx);
7800 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7801 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7802}
7803
7804
7805/**
7806 * Injects a double-fault (\#DF) exception into the VM.
7807 *
7808 * @returns Strict VBox status code (i.e. informational status codes too).
7809 * @param pVCpu The cross context virtual CPU structure.
7810 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7811 * out-of-sync. Make sure to update the required fields
7812 * before using them.
7813 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7814 * and should return VINF_EM_DBG_STEPPED if the event
7815 * is injected directly (register modified by us, not
7816 * by hardware on VM-entry).
7817 * @param puIntrState Pointer to the current guest interruptibility-state.
7818 * This interruptibility-state will be updated if
7819 * necessary. This cannot not be NULL.
7820 */
7821DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7822{
7823 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7824 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7825 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7826 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7827 fStepping, puIntrState);
7828}
7829
7830
7831/**
7832 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7833 *
7834 * @param pVCpu The cross context virtual CPU structure.
7835 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7836 * out-of-sync. Make sure to update the required fields
7837 * before using them.
7838 */
7839DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7840{
7841 NOREF(pMixedCtx);
7842 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7843 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7844 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7845}
7846
7847
7848/**
7849 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7850 *
7851 * @param pVCpu The cross context virtual CPU structure.
7852 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7853 * out-of-sync. Make sure to update the required fields
7854 * before using them.
7855 * @param cbInstr The value of RIP that is to be pushed on the guest
7856 * stack.
7857 */
7858DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7859{
7860 NOREF(pMixedCtx);
7861 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7862 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7863 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7864}
7865
7866
7867/**
7868 * Injects a general-protection (\#GP) fault into the VM.
7869 *
7870 * @returns Strict VBox status code (i.e. informational status codes too).
7871 * @param pVCpu The cross context virtual CPU structure.
7872 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7873 * out-of-sync. Make sure to update the required fields
7874 * before using them.
7875 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7876 * mode, i.e. in real-mode it's not valid).
7877 * @param u32ErrorCode The error code associated with the \#GP.
7878 * @param fStepping Whether we're running in
7879 * hmR0VmxRunGuestCodeStep() and should return
7880 * VINF_EM_DBG_STEPPED if the event is injected
7881 * directly (register modified by us, not by
7882 * hardware on VM-entry).
7883 * @param puIntrState Pointer to the current guest interruptibility-state.
7884 * This interruptibility-state will be updated if
7885 * necessary. This cannot not be NULL.
7886 */
7887DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7888 bool fStepping, uint32_t *puIntrState)
7889{
7890 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7891 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7892 if (fErrorCodeValid)
7893 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7894 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7895 fStepping, puIntrState);
7896}
7897
7898
7899#if 0 /* unused */
7900/**
7901 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7902 * VM.
7903 *
7904 * @param pVCpu The cross context virtual CPU structure.
7905 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7906 * out-of-sync. Make sure to update the required fields
7907 * before using them.
7908 * @param u32ErrorCode The error code associated with the \#GP.
7909 */
7910DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7911{
7912 NOREF(pMixedCtx);
7913 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7914 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7915 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7916 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7917}
7918#endif /* unused */
7919
7920
7921/**
7922 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7923 *
7924 * @param pVCpu The cross context virtual CPU structure.
7925 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7926 * out-of-sync. Make sure to update the required fields
7927 * before using them.
7928 * @param uVector The software interrupt vector number.
7929 * @param cbInstr The value of RIP that is to be pushed on the guest
7930 * stack.
7931 */
7932DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7933{
7934 NOREF(pMixedCtx);
7935 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7936 if ( uVector == X86_XCPT_BP
7937 || uVector == X86_XCPT_OF)
7938 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7939 else
7940 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7941 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7942}
7943
7944
7945/**
7946 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7947 * stack.
7948 *
7949 * @returns Strict VBox status code (i.e. informational status codes too).
7950 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7951 * @param pVM The cross context VM structure.
7952 * @param pMixedCtx Pointer to the guest-CPU context.
7953 * @param uValue The value to push to the guest stack.
7954 */
7955DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7956{
7957 /*
7958 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7959 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7960 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7961 */
7962 if (pMixedCtx->sp == 1)
7963 return VINF_EM_RESET;
7964 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7965 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7966 AssertRC(rc);
7967 return rc;
7968}
7969
7970
7971/**
7972 * Injects an event into the guest upon VM-entry by updating the relevant fields
7973 * in the VM-entry area in the VMCS.
7974 *
7975 * @returns Strict VBox status code (i.e. informational status codes too).
7976 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7977 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7978 *
7979 * @param pVCpu The cross context virtual CPU structure.
7980 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7981 * be out-of-sync. Make sure to update the required
7982 * fields before using them.
7983 * @param u64IntInfo The VM-entry interruption-information field.
7984 * @param cbInstr The VM-entry instruction length in bytes (for
7985 * software interrupts, exceptions and privileged
7986 * software exceptions).
7987 * @param u32ErrCode The VM-entry exception error code.
7988 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7989 * @param puIntrState Pointer to the current guest interruptibility-state.
7990 * This interruptibility-state will be updated if
7991 * necessary. This cannot not be NULL.
7992 * @param fStepping Whether we're running in
7993 * hmR0VmxRunGuestCodeStep() and should return
7994 * VINF_EM_DBG_STEPPED if the event is injected
7995 * directly (register modified by us, not by
7996 * hardware on VM-entry).
7997 *
7998 * @remarks Requires CR0!
7999 * @remarks No-long-jump zone!!!
8000 */
8001static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8002 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8003 uint32_t *puIntrState)
8004{
8005 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8006 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8007 Assert(puIntrState);
8008 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8009
8010 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8011 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8012
8013#ifdef VBOX_STRICT
8014 /* Validate the error-code-valid bit for hardware exceptions. */
8015 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
8016 {
8017 switch (uVector)
8018 {
8019 case X86_XCPT_PF:
8020 case X86_XCPT_DF:
8021 case X86_XCPT_TS:
8022 case X86_XCPT_NP:
8023 case X86_XCPT_SS:
8024 case X86_XCPT_GP:
8025 case X86_XCPT_AC:
8026 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8027 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8028 /* fallthru */
8029 default:
8030 break;
8031 }
8032 }
8033#endif
8034
8035 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8036 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8037 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8038
8039 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8040
8041 /* We require CR0 to check if the guest is in real-mode. */
8042 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8043 AssertRCReturn(rc, rc);
8044
8045 /*
8046 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8047 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8048 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8049 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8050 */
8051 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8052 {
8053 PVM pVM = pVCpu->CTX_SUFF(pVM);
8054 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8055 {
8056 Assert(PDMVmmDevHeapIsEnabled(pVM));
8057 Assert(pVM->hm.s.vmx.pRealModeTSS);
8058
8059 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8060 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8061 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8062 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8063 AssertRCReturn(rc, rc);
8064 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8065
8066 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8067 size_t const cbIdtEntry = sizeof(X86IDTR16);
8068 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8069 {
8070 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8071 if (uVector == X86_XCPT_DF)
8072 return VINF_EM_RESET;
8073
8074 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8075 if (uVector == X86_XCPT_GP)
8076 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8077
8078 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8079 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8080 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8081 fStepping, puIntrState);
8082 }
8083
8084 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8085 uint16_t uGuestIp = pMixedCtx->ip;
8086 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8087 {
8088 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8089 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8090 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8091 }
8092 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8093 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8094
8095 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8096 X86IDTR16 IdtEntry;
8097 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8098 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8099 AssertRCReturn(rc, rc);
8100
8101 /* Construct the stack frame for the interrupt/exception handler. */
8102 VBOXSTRICTRC rcStrict;
8103 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8104 if (rcStrict == VINF_SUCCESS)
8105 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8106 if (rcStrict == VINF_SUCCESS)
8107 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8108
8109 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8110 if (rcStrict == VINF_SUCCESS)
8111 {
8112 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8113 pMixedCtx->rip = IdtEntry.offSel;
8114 pMixedCtx->cs.Sel = IdtEntry.uSel;
8115 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8116 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8117 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8118 && uVector == X86_XCPT_PF)
8119 pMixedCtx->cr2 = GCPtrFaultAddress;
8120
8121 /* If any other guest-state bits are changed here, make sure to update
8122 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8124 | HM_CHANGED_GUEST_RIP
8125 | HM_CHANGED_GUEST_RFLAGS
8126 | HM_CHANGED_GUEST_RSP);
8127
8128 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8129 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8130 {
8131 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8132 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8133 Log4(("Clearing inhibition due to STI.\n"));
8134 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8135 }
8136 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8137 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8138
8139 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8140 it, if we are returning to ring-3 before executing guest code. */
8141 pVCpu->hm.s.Event.fPending = false;
8142
8143 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8144 if (fStepping)
8145 rcStrict = VINF_EM_DBG_STEPPED;
8146 }
8147 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8148 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8149 return rcStrict;
8150 }
8151
8152 /*
8153 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8154 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8155 */
8156 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8157 }
8158
8159 /* Validate. */
8160 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8161 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8162 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8163
8164 /* Inject. */
8165 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8166 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8167 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8168 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8169
8170 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8171 && uVector == X86_XCPT_PF)
8172 pMixedCtx->cr2 = GCPtrFaultAddress;
8173
8174 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8175 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8176
8177 AssertRCReturn(rc, rc);
8178 return VINF_SUCCESS;
8179}
8180
8181
8182/**
8183 * Clears the interrupt-window exiting control in the VMCS and if necessary
8184 * clears the current event in the VMCS as well.
8185 *
8186 * @returns VBox status code.
8187 * @param pVCpu The cross context virtual CPU structure.
8188 *
8189 * @remarks Use this function only to clear events that have not yet been
8190 * delivered to the guest but are injected in the VMCS!
8191 * @remarks No-long-jump zone!!!
8192 */
8193static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8194{
8195 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8196
8197 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8198 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8199
8200 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8201 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8202}
8203
8204
8205/**
8206 * Enters the VT-x session.
8207 *
8208 * @returns VBox status code.
8209 * @param pVM The cross context VM structure.
8210 * @param pVCpu The cross context virtual CPU structure.
8211 * @param pCpu Pointer to the CPU info struct.
8212 */
8213VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8214{
8215 AssertPtr(pVM);
8216 AssertPtr(pVCpu);
8217 Assert(pVM->hm.s.vmx.fSupported);
8218 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8219 NOREF(pCpu); NOREF(pVM);
8220
8221 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8222 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8223
8224#ifdef VBOX_STRICT
8225 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8226 RTCCUINTREG uHostCR4 = ASMGetCR4();
8227 if (!(uHostCR4 & X86_CR4_VMXE))
8228 {
8229 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8230 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8231 }
8232#endif
8233
8234 /*
8235 * Load the VCPU's VMCS as the current (and active) one.
8236 */
8237 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8238 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8239 if (RT_FAILURE(rc))
8240 return rc;
8241
8242 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8243 pVCpu->hm.s.fLeaveDone = false;
8244 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8245
8246 return VINF_SUCCESS;
8247}
8248
8249
8250/**
8251 * The thread-context callback (only on platforms which support it).
8252 *
8253 * @param enmEvent The thread-context event.
8254 * @param pVCpu The cross context virtual CPU structure.
8255 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8256 * @thread EMT(pVCpu)
8257 */
8258VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8259{
8260 NOREF(fGlobalInit);
8261
8262 switch (enmEvent)
8263 {
8264 case RTTHREADCTXEVENT_OUT:
8265 {
8266 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8267 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8268 VMCPU_ASSERT_EMT(pVCpu);
8269
8270 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8271
8272 /* No longjmps (logger flushes, locks) in this fragile context. */
8273 VMMRZCallRing3Disable(pVCpu);
8274 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8275
8276 /*
8277 * Restore host-state (FPU, debug etc.)
8278 */
8279 if (!pVCpu->hm.s.fLeaveDone)
8280 {
8281 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8282 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8283 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8284 pVCpu->hm.s.fLeaveDone = true;
8285 }
8286
8287 /* Leave HM context, takes care of local init (term). */
8288 int rc = HMR0LeaveCpu(pVCpu);
8289 AssertRC(rc); NOREF(rc);
8290
8291 /* Restore longjmp state. */
8292 VMMRZCallRing3Enable(pVCpu);
8293 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8294 break;
8295 }
8296
8297 case RTTHREADCTXEVENT_IN:
8298 {
8299 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8300 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8301 VMCPU_ASSERT_EMT(pVCpu);
8302
8303 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8304 VMMRZCallRing3Disable(pVCpu);
8305 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8306
8307 /* Initialize the bare minimum state required for HM. This takes care of
8308 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8309 int rc = HMR0EnterCpu(pVCpu);
8310 AssertRC(rc);
8311 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8312
8313 /* Load the active VMCS as the current one. */
8314 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8315 {
8316 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8317 AssertRC(rc); NOREF(rc);
8318 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8319 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8320 }
8321 pVCpu->hm.s.fLeaveDone = false;
8322
8323 /* Restore longjmp state. */
8324 VMMRZCallRing3Enable(pVCpu);
8325 break;
8326 }
8327
8328 default:
8329 break;
8330 }
8331}
8332
8333
8334/**
8335 * Saves the host state in the VMCS host-state.
8336 * Sets up the VM-exit MSR-load area.
8337 *
8338 * The CPU state will be loaded from these fields on every successful VM-exit.
8339 *
8340 * @returns VBox status code.
8341 * @param pVM The cross context VM structure.
8342 * @param pVCpu The cross context virtual CPU structure.
8343 *
8344 * @remarks No-long-jump zone!!!
8345 */
8346static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8347{
8348 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8349
8350 int rc = VINF_SUCCESS;
8351 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8352 {
8353 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8354 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8355
8356 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8357 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8358
8359 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8360 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8361
8362 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8363 }
8364 return rc;
8365}
8366
8367
8368/**
8369 * Saves the host state in the VMCS host-state.
8370 *
8371 * @returns VBox status code.
8372 * @param pVM The cross context VM structure.
8373 * @param pVCpu The cross context virtual CPU structure.
8374 *
8375 * @remarks No-long-jump zone!!!
8376 */
8377VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8378{
8379 AssertPtr(pVM);
8380 AssertPtr(pVCpu);
8381
8382 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8383
8384 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8385 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8386 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8387 return hmR0VmxSaveHostState(pVM, pVCpu);
8388}
8389
8390
8391/**
8392 * Loads the guest state into the VMCS guest-state area.
8393 *
8394 * The will typically be done before VM-entry when the guest-CPU state and the
8395 * VMCS state may potentially be out of sync.
8396 *
8397 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8398 * VM-entry controls.
8399 * Sets up the appropriate VMX non-root function to execute guest code based on
8400 * the guest CPU mode.
8401 *
8402 * @returns VBox strict status code.
8403 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8404 * without unrestricted guest access and the VMMDev is not presently
8405 * mapped (e.g. EFI32).
8406 *
8407 * @param pVM The cross context VM structure.
8408 * @param pVCpu The cross context virtual CPU structure.
8409 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8410 * out-of-sync. Make sure to update the required fields
8411 * before using them.
8412 */
8413static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8414{
8415 AssertPtr(pVM);
8416 AssertPtr(pVCpu);
8417 AssertPtr(pMixedCtx);
8418 HMVMX_ASSERT_PREEMPT_SAFE();
8419
8420 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8421
8422 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8423
8424 /* Determine real-on-v86 mode. */
8425 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8426 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8427 && CPUMIsGuestInRealModeEx(pMixedCtx))
8428 {
8429 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8430 }
8431
8432 /*
8433 * Load the guest-state into the VMCS.
8434 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8435 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8436 */
8437 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8438 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8439
8440 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8441 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8442 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8443
8444 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8445 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8446 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8447
8448 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8449 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8450
8451 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8452 if (rcStrict == VINF_SUCCESS)
8453 { /* likely */ }
8454 else
8455 {
8456 VMMRZCallRing3Enable(pVCpu);
8457 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8458 return rcStrict;
8459 }
8460
8461 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8462 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8463 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8464
8465 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8466 determine we don't have to swap EFER after all. */
8467 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8468 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8469
8470 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8471 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8472
8473 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8474 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8475
8476 /*
8477 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8478 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8479 */
8480 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8481 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8482
8483 /* Clear any unused and reserved bits. */
8484 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8485
8486 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8487 return rc;
8488}
8489
8490
8491/**
8492 * Loads the state shared between the host and guest into the VMCS.
8493 *
8494 * @param pVM The cross context VM structure.
8495 * @param pVCpu The cross context virtual CPU structure.
8496 * @param pCtx Pointer to the guest-CPU context.
8497 *
8498 * @remarks No-long-jump zone!!!
8499 */
8500static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8501{
8502 NOREF(pVM);
8503
8504 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8505 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8506
8507 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8508 {
8509 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8510 AssertRC(rc);
8511 }
8512
8513 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8514 {
8515 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8516 AssertRC(rc);
8517
8518 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8519 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8520 {
8521 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8522 AssertRC(rc);
8523 }
8524 }
8525
8526 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8527 {
8528 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8529 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8530 }
8531
8532 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8534 {
8535 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8536 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8537 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8538 AssertRC(rc);
8539 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8540 }
8541
8542 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8543 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8544}
8545
8546
8547/**
8548 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8549 *
8550 * @returns Strict VBox status code (i.e. informational status codes too).
8551 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8552 * without unrestricted guest access and the VMMDev is not presently
8553 * mapped (e.g. EFI32).
8554 *
8555 * @param pVM The cross context VM structure.
8556 * @param pVCpu The cross context virtual CPU structure.
8557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8558 * out-of-sync. Make sure to update the required fields
8559 * before using them.
8560 */
8561static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8562{
8563 HMVMX_ASSERT_PREEMPT_SAFE();
8564
8565 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8566#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8567 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8568#endif
8569
8570 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8571 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8572 {
8573 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8574 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8575 { /* likely */}
8576 else
8577 {
8578 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8579 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8580 }
8581 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8582 }
8583 else if (HMCPU_CF_VALUE(pVCpu))
8584 {
8585 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8586 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8587 { /* likely */}
8588 else
8589 {
8590 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8591 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8592 return rcStrict;
8593 }
8594 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8595 }
8596
8597 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8598 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8599 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8600 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8601 return rcStrict;
8602}
8603
8604
8605/**
8606 * Does the preparations before executing guest code in VT-x.
8607 *
8608 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8609 * recompiler/IEM. We must be cautious what we do here regarding committing
8610 * guest-state information into the VMCS assuming we assuredly execute the
8611 * guest in VT-x mode.
8612 *
8613 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8614 * the common-state (TRPM/forceflags), we must undo those changes so that the
8615 * recompiler/IEM can (and should) use them when it resumes guest execution.
8616 * Otherwise such operations must be done when we can no longer exit to ring-3.
8617 *
8618 * @returns Strict VBox status code (i.e. informational status codes too).
8619 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8620 * have been disabled.
8621 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8622 * double-fault into the guest.
8623 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8624 * dispatched directly.
8625 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8626 *
8627 * @param pVM The cross context VM structure.
8628 * @param pVCpu The cross context virtual CPU structure.
8629 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8630 * out-of-sync. Make sure to update the required fields
8631 * before using them.
8632 * @param pVmxTransient Pointer to the VMX transient structure.
8633 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8634 * us ignore some of the reasons for returning to
8635 * ring-3, and return VINF_EM_DBG_STEPPED if event
8636 * dispatching took place.
8637 */
8638static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8639{
8640 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8641
8642#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8643 PGMRZDynMapFlushAutoSet(pVCpu);
8644#endif
8645
8646 /* Check force flag actions that might require us to go back to ring-3. */
8647 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8648 if (rcStrict == VINF_SUCCESS)
8649 { /* FFs doesn't get set all the time. */ }
8650 else
8651 return rcStrict;
8652
8653 if (TRPMHasTrap(pVCpu))
8654 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8655 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8656
8657 /*
8658 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8659 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8660 */
8661 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8662 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8663 { /* likely */ }
8664 else
8665 {
8666 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8667 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8668 return rcStrict;
8669 }
8670
8671 /*
8672 * Load the guest state bits, we can handle longjmps/getting preempted here.
8673 *
8674 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8675 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8676 * Hence, this needs to be done -after- injection of events.
8677 */
8678 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8679 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8680 { /* likely */ }
8681 else
8682 return rcStrict;
8683
8684 /*
8685 * No longjmps to ring-3 from this point on!!!
8686 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8687 * This also disables flushing of the R0-logger instance (if any).
8688 */
8689 VMMRZCallRing3Disable(pVCpu);
8690
8691 /*
8692 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8693 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8694 *
8695 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8696 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8697 *
8698 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8699 * executing guest code.
8700 */
8701 pVmxTransient->fEFlags = ASMIntDisableFlags();
8702
8703 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8704 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8705 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8706 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8707 {
8708 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8709 {
8710 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8711 pVCpu->hm.s.Event.fPending = false;
8712
8713 return VINF_SUCCESS;
8714 }
8715
8716 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8717 rcStrict = VINF_EM_RAW_INTERRUPT;
8718 }
8719 else
8720 {
8721 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8722 rcStrict = VINF_EM_RAW_TO_R3;
8723 }
8724
8725 ASMSetFlags(pVmxTransient->fEFlags);
8726 VMMRZCallRing3Enable(pVCpu);
8727
8728 return rcStrict;
8729}
8730
8731
8732/**
8733 * Prepares to run guest code in VT-x and we've committed to doing so. This
8734 * means there is no backing out to ring-3 or anywhere else at this
8735 * point.
8736 *
8737 * @param pVM The cross context VM structure.
8738 * @param pVCpu The cross context virtual CPU structure.
8739 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8740 * out-of-sync. Make sure to update the required fields
8741 * before using them.
8742 * @param pVmxTransient Pointer to the VMX transient structure.
8743 *
8744 * @remarks Called with preemption disabled.
8745 * @remarks No-long-jump zone!!!
8746 */
8747static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8748{
8749 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8750 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8751 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8752
8753 /*
8754 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8755 */
8756 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8757 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8758
8759#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8760 if (!CPUMIsGuestFPUStateActive(pVCpu))
8761 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8762 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8763 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8764#endif
8765
8766 if ( pVCpu->hm.s.fPreloadGuestFpu
8767 && !CPUMIsGuestFPUStateActive(pVCpu))
8768 {
8769 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8770 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8771 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8772 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8773 }
8774
8775 /*
8776 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8777 */
8778 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8779 && pVCpu->hm.s.vmx.cMsrs > 0)
8780 {
8781 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8782 }
8783
8784 /*
8785 * Load the host state bits as we may've been preempted (only happens when
8786 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8787 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8788 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8789 * See @bugref{8432}.
8790 */
8791 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8792 {
8793 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8794 AssertRC(rc);
8795 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8796 }
8797 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8798
8799 /*
8800 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8801 */
8802 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8803 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8804 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8805
8806 /* Store status of the shared guest-host state at the time of VM-entry. */
8807#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8808 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8809 {
8810 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8811 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8812 }
8813 else
8814#endif
8815 {
8816 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8817 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8818 }
8819 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8820
8821 /*
8822 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8823 */
8824 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8825 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8826
8827 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8828 RTCPUID idCurrentCpu = pCpu->idCpu;
8829 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8830 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8831 {
8832 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8833 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8834 }
8835
8836 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8837 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8838 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8839 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8840
8841 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8842
8843 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8844 to start executing. */
8845
8846 /*
8847 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8848 */
8849 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8850 {
8851 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8852 {
8853 bool fMsrUpdated;
8854 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8855 AssertRC(rc2);
8856 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8857
8858 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8859 &fMsrUpdated);
8860 AssertRC(rc2);
8861 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8862
8863 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8864 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8865 }
8866 else
8867 {
8868 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8869 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8870 }
8871 }
8872
8873#ifdef VBOX_STRICT
8874 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8875 hmR0VmxCheckHostEferMsr(pVCpu);
8876 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8877#endif
8878#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8879 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8880 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8881 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8882#endif
8883}
8884
8885
8886/**
8887 * Performs some essential restoration of state after running guest code in
8888 * VT-x.
8889 *
8890 * @param pVM The cross context VM structure.
8891 * @param pVCpu The cross context virtual CPU structure.
8892 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8893 * out-of-sync. Make sure to update the required fields
8894 * before using them.
8895 * @param pVmxTransient Pointer to the VMX transient structure.
8896 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8897 *
8898 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8899 *
8900 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8901 * unconditionally when it is safe to do so.
8902 */
8903static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8904{
8905 NOREF(pVM);
8906
8907 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8908
8909 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8910 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8911 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8912 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8913 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8914 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8915
8916 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8917 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8918
8919 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8920 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8921 Assert(!ASMIntAreEnabled());
8922 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8923
8924#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8925 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8926 {
8927 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8928 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8929 }
8930#endif
8931
8932#if HC_ARCH_BITS == 64
8933 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8934#endif
8935#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8936 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8937 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8938 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8939#else
8940 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8941#endif
8942#ifdef VBOX_STRICT
8943 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8944#endif
8945 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8946 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8947
8948 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8949 uint32_t uExitReason;
8950 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8951 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8952 AssertRC(rc);
8953 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8954 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8955
8956 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8957 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8958 {
8959 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8960 pVmxTransient->fVMEntryFailed));
8961 return;
8962 }
8963
8964 /*
8965 * Update the VM-exit history array here even if the VM-entry failed due to:
8966 * - Invalid guest state.
8967 * - MSR loading.
8968 * - Machine-check event.
8969 *
8970 * In any of the above cases we will still have a "valid" VM-exit reason
8971 * despite @a fVMEntryFailed being false.
8972 *
8973 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8974 */
8975 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8976
8977 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8978 {
8979 /** @todo We can optimize this by only syncing with our force-flags when
8980 * really needed and keeping the VMCS state as it is for most
8981 * VM-exits. */
8982 /* Update the guest interruptibility-state from the VMCS. */
8983 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8984
8985#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8986 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8987 AssertRC(rc);
8988#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8989 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8990 AssertRC(rc);
8991#endif
8992
8993 /*
8994 * Sync the TPR shadow with our APIC state.
8995 */
8996 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8997 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8998 {
8999 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9000 AssertRC(rc);
9001 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
9002 }
9003 }
9004}
9005
9006
9007/**
9008 * Runs the guest code using VT-x the normal way.
9009 *
9010 * @returns VBox status code.
9011 * @param pVM The cross context VM structure.
9012 * @param pVCpu The cross context virtual CPU structure.
9013 * @param pCtx Pointer to the guest-CPU context.
9014 *
9015 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9016 */
9017static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9018{
9019 VMXTRANSIENT VmxTransient;
9020 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9021 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9022 uint32_t cLoops = 0;
9023
9024 for (;; cLoops++)
9025 {
9026 Assert(!HMR0SuspendPending());
9027 HMVMX_ASSERT_CPU_SAFE();
9028
9029 /* Preparatory work for running guest code, this may force us to return
9030 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9031 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9032 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9033 if (rcStrict != VINF_SUCCESS)
9034 break;
9035
9036 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9037 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9038 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9039
9040 /* Restore any residual host-state and save any bits shared between host
9041 and guest into the guest-CPU state. Re-enables interrupts! */
9042 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9043
9044 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9045 if (RT_SUCCESS(rcRun))
9046 { /* very likely */ }
9047 else
9048 {
9049 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9050 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9051 return rcRun;
9052 }
9053
9054 /* Profile the VM-exit. */
9055 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9056 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9057 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9058 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9059 HMVMX_START_EXIT_DISPATCH_PROF();
9060
9061 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9062
9063 /* Handle the VM-exit. */
9064#ifdef HMVMX_USE_FUNCTION_TABLE
9065 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9066#else
9067 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9068#endif
9069 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9070 if (rcStrict == VINF_SUCCESS)
9071 {
9072 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9073 continue; /* likely */
9074 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9075 rcStrict = VINF_EM_RAW_INTERRUPT;
9076 }
9077 break;
9078 }
9079
9080 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9081 return rcStrict;
9082}
9083
9084
9085
9086/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9087 * probes.
9088 *
9089 * The following few functions and associated structure contains the bloat
9090 * necessary for providing detailed debug events and dtrace probes as well as
9091 * reliable host side single stepping. This works on the principle of
9092 * "subclassing" the normal execution loop and workers. We replace the loop
9093 * method completely and override selected helpers to add necessary adjustments
9094 * to their core operation.
9095 *
9096 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9097 * any performance for debug and analysis features.
9098 *
9099 * @{
9100 */
9101
9102/**
9103 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9104 * the debug run loop.
9105 */
9106typedef struct VMXRUNDBGSTATE
9107{
9108 /** The RIP we started executing at. This is for detecting that we stepped. */
9109 uint64_t uRipStart;
9110 /** The CS we started executing with. */
9111 uint16_t uCsStart;
9112
9113 /** Whether we've actually modified the 1st execution control field. */
9114 bool fModifiedProcCtls : 1;
9115 /** Whether we've actually modified the 2nd execution control field. */
9116 bool fModifiedProcCtls2 : 1;
9117 /** Whether we've actually modified the exception bitmap. */
9118 bool fModifiedXcptBitmap : 1;
9119
9120 /** We desire the modified the CR0 mask to be cleared. */
9121 bool fClearCr0Mask : 1;
9122 /** We desire the modified the CR4 mask to be cleared. */
9123 bool fClearCr4Mask : 1;
9124 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9125 uint32_t fCpe1Extra;
9126 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9127 uint32_t fCpe1Unwanted;
9128 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9129 uint32_t fCpe2Extra;
9130 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9131 uint32_t bmXcptExtra;
9132 /** The sequence number of the Dtrace provider settings the state was
9133 * configured against. */
9134 uint32_t uDtraceSettingsSeqNo;
9135 /** VM-exits to check (one bit per VM-exit). */
9136 uint32_t bmExitsToCheck[3];
9137
9138 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9139 uint32_t fProcCtlsInitial;
9140 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9141 uint32_t fProcCtls2Initial;
9142 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9143 uint32_t bmXcptInitial;
9144} VMXRUNDBGSTATE;
9145AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9146typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9147
9148
9149/**
9150 * Initializes the VMXRUNDBGSTATE structure.
9151 *
9152 * @param pVCpu The cross context virtual CPU structure of the
9153 * calling EMT.
9154 * @param pCtx The CPU register context to go with @a pVCpu.
9155 * @param pDbgState The structure to initialize.
9156 */
9157DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9158{
9159 pDbgState->uRipStart = pCtx->rip;
9160 pDbgState->uCsStart = pCtx->cs.Sel;
9161
9162 pDbgState->fModifiedProcCtls = false;
9163 pDbgState->fModifiedProcCtls2 = false;
9164 pDbgState->fModifiedXcptBitmap = false;
9165 pDbgState->fClearCr0Mask = false;
9166 pDbgState->fClearCr4Mask = false;
9167 pDbgState->fCpe1Extra = 0;
9168 pDbgState->fCpe1Unwanted = 0;
9169 pDbgState->fCpe2Extra = 0;
9170 pDbgState->bmXcptExtra = 0;
9171 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9172 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9173 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9174}
9175
9176
9177/**
9178 * Updates the VMSC fields with changes requested by @a pDbgState.
9179 *
9180 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9181 * immediately before executing guest code, i.e. when interrupts are disabled.
9182 * We don't check status codes here as we cannot easily assert or return in the
9183 * latter case.
9184 *
9185 * @param pVCpu The cross context virtual CPU structure.
9186 * @param pDbgState The debug state.
9187 */
9188DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9189{
9190 /*
9191 * Ensure desired flags in VMCS control fields are set.
9192 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9193 *
9194 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9195 * there should be no stale data in pCtx at this point.
9196 */
9197 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9198 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9199 {
9200 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9201 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9202 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9203 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9204 pDbgState->fModifiedProcCtls = true;
9205 }
9206
9207 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9208 {
9209 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9210 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9211 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9212 pDbgState->fModifiedProcCtls2 = true;
9213 }
9214
9215 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9216 {
9217 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9218 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9219 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9220 pDbgState->fModifiedXcptBitmap = true;
9221 }
9222
9223 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9224 {
9225 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9226 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9227 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9228 }
9229
9230 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9231 {
9232 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9233 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9234 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9235 }
9236}
9237
9238
9239DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9240{
9241 /*
9242 * Restore VM-exit control settings as we may not reenter this function the
9243 * next time around.
9244 */
9245 /* We reload the initial value, trigger what we can of recalculations the
9246 next time around. From the looks of things, that's all that's required atm. */
9247 if (pDbgState->fModifiedProcCtls)
9248 {
9249 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9250 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9251 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9252 AssertRCReturn(rc2, rc2);
9253 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9254 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9255 }
9256
9257 /* We're currently the only ones messing with this one, so just restore the
9258 cached value and reload the field. */
9259 if ( pDbgState->fModifiedProcCtls2
9260 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9261 {
9262 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9263 AssertRCReturn(rc2, rc2);
9264 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9265 }
9266
9267 /* If we've modified the exception bitmap, we restore it and trigger
9268 reloading and partial recalculation the next time around. */
9269 if (pDbgState->fModifiedXcptBitmap)
9270 {
9271 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9272 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9273 }
9274
9275 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9276 if (pDbgState->fClearCr0Mask)
9277 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9278
9279 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9280 if (pDbgState->fClearCr4Mask)
9281 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9282
9283 return rcStrict;
9284}
9285
9286
9287/**
9288 * Configures VM-exit controls for current DBGF and DTrace settings.
9289 *
9290 * This updates @a pDbgState and the VMCS execution control fields to reflect
9291 * the necessary VM-exits demanded by DBGF and DTrace.
9292 *
9293 * @param pVM The cross context VM structure.
9294 * @param pVCpu The cross context virtual CPU structure.
9295 * @param pCtx Pointer to the guest-CPU context.
9296 * @param pDbgState The debug state.
9297 * @param pVmxTransient Pointer to the VMX transient structure. May update
9298 * fUpdateTscOffsettingAndPreemptTimer.
9299 */
9300static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9301 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9302{
9303 /*
9304 * Take down the dtrace serial number so we can spot changes.
9305 */
9306 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9307 ASMCompilerBarrier();
9308
9309 /*
9310 * We'll rebuild most of the middle block of data members (holding the
9311 * current settings) as we go along here, so start by clearing it all.
9312 */
9313 pDbgState->bmXcptExtra = 0;
9314 pDbgState->fCpe1Extra = 0;
9315 pDbgState->fCpe1Unwanted = 0;
9316 pDbgState->fCpe2Extra = 0;
9317 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9318 pDbgState->bmExitsToCheck[i] = 0;
9319
9320 /*
9321 * Software interrupts (INT XXh) - no idea how to trigger these...
9322 */
9323 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9324 || VBOXVMM_INT_SOFTWARE_ENABLED())
9325 {
9326 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9327 }
9328
9329 /*
9330 * INT3 breakpoints - triggered by #BP exceptions.
9331 */
9332 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9333 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9334
9335 /*
9336 * Exception bitmap and XCPT events+probes.
9337 */
9338 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9339 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9340 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9341
9342 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9343 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9344 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9345 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9346 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9347 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9348 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9349 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9350 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9351 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9352 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9353 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9354 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9355 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9356 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9357 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9358 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9359 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9360
9361 if (pDbgState->bmXcptExtra)
9362 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9363
9364 /*
9365 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9366 *
9367 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9368 * So, when adding/changing/removing please don't forget to update it.
9369 *
9370 * Some of the macros are picking up local variables to save horizontal space,
9371 * (being able to see it in a table is the lesser evil here).
9372 */
9373#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9374 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9375 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9376#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9377 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9378 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9379 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9380 } else do { } while (0)
9381#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9382 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9383 { \
9384 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9385 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9386 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9387 } else do { } while (0)
9388#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9389 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9390 { \
9391 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9392 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9393 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9394 } else do { } while (0)
9395#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9396 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9397 { \
9398 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9399 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9400 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9401 } else do { } while (0)
9402
9403 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9404 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9405 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9406 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9407 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9408
9409 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9411 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9412 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9413 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9415 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9417 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9419 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9421 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9423 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9425 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9429 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9431 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9432 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9433 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9435 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9436 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9437 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9438 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9439 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9440 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9441 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9442 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9443 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9444 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9445
9446 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9447 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9448 {
9449 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9450 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9451 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9452 AssertRC(rc2);
9453
9454#if 0 /** @todo fix me */
9455 pDbgState->fClearCr0Mask = true;
9456 pDbgState->fClearCr4Mask = true;
9457#endif
9458 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9459 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9460 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9461 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9462 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9463 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9464 require clearing here and in the loop if we start using it. */
9465 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9466 }
9467 else
9468 {
9469 if (pDbgState->fClearCr0Mask)
9470 {
9471 pDbgState->fClearCr0Mask = false;
9472 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9473 }
9474 if (pDbgState->fClearCr4Mask)
9475 {
9476 pDbgState->fClearCr4Mask = false;
9477 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9478 }
9479 }
9480 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9481 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9482
9483 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9484 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9485 {
9486 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9487 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9488 }
9489 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9490 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9491
9492 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9493 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9494 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9495 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9496 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9497 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9498 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9499 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9500#if 0 /** @todo too slow, fix handler. */
9501 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9502#endif
9503 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9504
9505 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9506 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9507 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9508 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9509 {
9510 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9511 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9512 }
9513 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9514 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9515 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9516 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9517
9518 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9519 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9520 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9521 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9522 {
9523 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9524 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9525 }
9526 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9527 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9528 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9529 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9530
9531 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9532 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9533 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9534 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9535 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9536 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9537 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9538 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9539 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9540 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9541 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9542 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9543 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9544 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9545 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9546 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9547 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9548 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9549 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9550 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9551 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9552 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9553
9554#undef IS_EITHER_ENABLED
9555#undef SET_ONLY_XBM_IF_EITHER_EN
9556#undef SET_CPE1_XBM_IF_EITHER_EN
9557#undef SET_CPEU_XBM_IF_EITHER_EN
9558#undef SET_CPE2_XBM_IF_EITHER_EN
9559
9560 /*
9561 * Sanitize the control stuff.
9562 */
9563 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9564 if (pDbgState->fCpe2Extra)
9565 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9566 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9567 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9568 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9569 {
9570 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9571 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9572 }
9573
9574 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9575 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9576 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9577 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9578}
9579
9580
9581/**
9582 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9583 * appropriate.
9584 *
9585 * The caller has checked the VM-exit against the
9586 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9587 * already, so we don't have to do that either.
9588 *
9589 * @returns Strict VBox status code (i.e. informational status codes too).
9590 * @param pVM The cross context VM structure.
9591 * @param pVCpu The cross context virtual CPU structure.
9592 * @param pMixedCtx Pointer to the guest-CPU context.
9593 * @param pVmxTransient Pointer to the VMX-transient structure.
9594 * @param uExitReason The VM-exit reason.
9595 *
9596 * @remarks The name of this function is displayed by dtrace, so keep it short
9597 * and to the point. No longer than 33 chars long, please.
9598 */
9599static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9600 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9601{
9602 /*
9603 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9604 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9605 *
9606 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9607 * does. Must add/change/remove both places. Same ordering, please.
9608 *
9609 * Added/removed events must also be reflected in the next section
9610 * where we dispatch dtrace events.
9611 */
9612 bool fDtrace1 = false;
9613 bool fDtrace2 = false;
9614 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9615 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9616 uint32_t uEventArg = 0;
9617#define SET_EXIT(a_EventSubName) \
9618 do { \
9619 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9620 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9621 } while (0)
9622#define SET_BOTH(a_EventSubName) \
9623 do { \
9624 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9625 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9626 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9627 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9628 } while (0)
9629 switch (uExitReason)
9630 {
9631 case VMX_EXIT_MTF:
9632 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9633
9634 case VMX_EXIT_XCPT_OR_NMI:
9635 {
9636 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9637 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9638 {
9639 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9640 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9641 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9642 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9643 {
9644 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9645 {
9646 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9647 uEventArg = pVmxTransient->uExitIntErrorCode;
9648 }
9649 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9650 switch (enmEvent1)
9651 {
9652 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9653 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9654 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9655 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9656 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9657 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9658 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9659 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9660 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9661 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9662 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9663 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9664 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9665 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9666 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9667 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9668 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9669 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9670 default: break;
9671 }
9672 }
9673 else
9674 AssertFailed();
9675 break;
9676
9677 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9678 uEventArg = idxVector;
9679 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9680 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9681 break;
9682 }
9683 break;
9684 }
9685
9686 case VMX_EXIT_TRIPLE_FAULT:
9687 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9688 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9689 break;
9690 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9691 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9692 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9693 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9694 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9695
9696 /* Instruction specific VM-exits: */
9697 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9698 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9699 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9700 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9701 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9702 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9703 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9704 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9705 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9706 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9707 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9708 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9709 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9710 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9711 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9712 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9713 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9714 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9715 case VMX_EXIT_MOV_CRX:
9716 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9717/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9718* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9719 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9720 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9721 SET_BOTH(CRX_READ);
9722 else
9723 SET_BOTH(CRX_WRITE);
9724 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9725 break;
9726 case VMX_EXIT_MOV_DRX:
9727 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9728 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9729 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9730 SET_BOTH(DRX_READ);
9731 else
9732 SET_BOTH(DRX_WRITE);
9733 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9734 break;
9735 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9736 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9737 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9738 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9739 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9740 case VMX_EXIT_XDTR_ACCESS:
9741 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9742 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9743 {
9744 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9745 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9746 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9747 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9748 }
9749 break;
9750
9751 case VMX_EXIT_TR_ACCESS:
9752 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9753 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9754 {
9755 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9756 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9757 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9758 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9759 }
9760 break;
9761
9762 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9763 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9764 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9765 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9766 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9767 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9768 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9769 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9770 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9771 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9772 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9773
9774 /* Events that aren't relevant at this point. */
9775 case VMX_EXIT_EXT_INT:
9776 case VMX_EXIT_INT_WINDOW:
9777 case VMX_EXIT_NMI_WINDOW:
9778 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9779 case VMX_EXIT_PREEMPT_TIMER:
9780 case VMX_EXIT_IO_INSTR:
9781 break;
9782
9783 /* Errors and unexpected events. */
9784 case VMX_EXIT_INIT_SIGNAL:
9785 case VMX_EXIT_SIPI:
9786 case VMX_EXIT_IO_SMI:
9787 case VMX_EXIT_SMI:
9788 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9789 case VMX_EXIT_ERR_MSR_LOAD:
9790 case VMX_EXIT_ERR_MACHINE_CHECK:
9791 break;
9792
9793 default:
9794 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9795 break;
9796 }
9797#undef SET_BOTH
9798#undef SET_EXIT
9799
9800 /*
9801 * Dtrace tracepoints go first. We do them here at once so we don't
9802 * have to copy the guest state saving and stuff a few dozen times.
9803 * Down side is that we've got to repeat the switch, though this time
9804 * we use enmEvent since the probes are a subset of what DBGF does.
9805 */
9806 if (fDtrace1 || fDtrace2)
9807 {
9808 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9809 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9810 switch (enmEvent1)
9811 {
9812 /** @todo consider which extra parameters would be helpful for each probe. */
9813 case DBGFEVENT_END: break;
9814 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9816 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9820 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9821 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9822 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9823 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9824 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9825 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9826 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9827 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9829 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9830 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9831 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9832 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9833 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9834 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9836 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9839 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9840 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9841 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9842 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9843 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9844 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9845 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9846 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9847 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9848 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9877 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9878 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9879 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9880 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9881 }
9882 switch (enmEvent2)
9883 {
9884 /** @todo consider which extra parameters would be helpful for each probe. */
9885 case DBGFEVENT_END: break;
9886 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9887 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9888 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9893 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9894 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9895 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9896 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9897 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9898 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9899 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9900 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9901 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9902 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9903 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9904 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9905 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9906 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9908 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9929 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9930 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9931 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9932 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9933 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9934 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9935 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9936 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9937 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9938 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9939 }
9940 }
9941
9942 /*
9943 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9944 * the DBGF call will do a full check).
9945 *
9946 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9947 * Note! If we have to events, we prioritize the first, i.e. the instruction
9948 * one, in order to avoid event nesting.
9949 */
9950 if ( enmEvent1 != DBGFEVENT_END
9951 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9952 {
9953 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9954 if (rcStrict != VINF_SUCCESS)
9955 return rcStrict;
9956 }
9957 else if ( enmEvent2 != DBGFEVENT_END
9958 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9959 {
9960 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9961 if (rcStrict != VINF_SUCCESS)
9962 return rcStrict;
9963 }
9964
9965 return VINF_SUCCESS;
9966}
9967
9968
9969/**
9970 * Single-stepping VM-exit filtering.
9971 *
9972 * This is preprocessing the VM-exits and deciding whether we've gotten far
9973 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9974 * handling is performed.
9975 *
9976 * @returns Strict VBox status code (i.e. informational status codes too).
9977 * @param pVM The cross context VM structure.
9978 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9979 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9980 * out-of-sync. Make sure to update the required
9981 * fields before using them.
9982 * @param pVmxTransient Pointer to the VMX-transient structure.
9983 * @param uExitReason The VM-exit reason.
9984 * @param pDbgState The debug state.
9985 */
9986DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9987 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9988{
9989 /*
9990 * Expensive (saves context) generic dtrace VM-exit probe.
9991 */
9992 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9993 { /* more likely */ }
9994 else
9995 {
9996 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9997 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9998 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9999 }
10000
10001 /*
10002 * Check for host NMI, just to get that out of the way.
10003 */
10004 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10005 { /* normally likely */ }
10006 else
10007 {
10008 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10009 AssertRCReturn(rc2, rc2);
10010 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10011 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10012 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10013 }
10014
10015 /*
10016 * Check for single stepping event if we're stepping.
10017 */
10018 if (pVCpu->hm.s.fSingleInstruction)
10019 {
10020 switch (uExitReason)
10021 {
10022 case VMX_EXIT_MTF:
10023 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10024
10025 /* Various events: */
10026 case VMX_EXIT_XCPT_OR_NMI:
10027 case VMX_EXIT_EXT_INT:
10028 case VMX_EXIT_TRIPLE_FAULT:
10029 case VMX_EXIT_INT_WINDOW:
10030 case VMX_EXIT_NMI_WINDOW:
10031 case VMX_EXIT_TASK_SWITCH:
10032 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10033 case VMX_EXIT_APIC_ACCESS:
10034 case VMX_EXIT_EPT_VIOLATION:
10035 case VMX_EXIT_EPT_MISCONFIG:
10036 case VMX_EXIT_PREEMPT_TIMER:
10037
10038 /* Instruction specific VM-exits: */
10039 case VMX_EXIT_CPUID:
10040 case VMX_EXIT_GETSEC:
10041 case VMX_EXIT_HLT:
10042 case VMX_EXIT_INVD:
10043 case VMX_EXIT_INVLPG:
10044 case VMX_EXIT_RDPMC:
10045 case VMX_EXIT_RDTSC:
10046 case VMX_EXIT_RSM:
10047 case VMX_EXIT_VMCALL:
10048 case VMX_EXIT_VMCLEAR:
10049 case VMX_EXIT_VMLAUNCH:
10050 case VMX_EXIT_VMPTRLD:
10051 case VMX_EXIT_VMPTRST:
10052 case VMX_EXIT_VMREAD:
10053 case VMX_EXIT_VMRESUME:
10054 case VMX_EXIT_VMWRITE:
10055 case VMX_EXIT_VMXOFF:
10056 case VMX_EXIT_VMXON:
10057 case VMX_EXIT_MOV_CRX:
10058 case VMX_EXIT_MOV_DRX:
10059 case VMX_EXIT_IO_INSTR:
10060 case VMX_EXIT_RDMSR:
10061 case VMX_EXIT_WRMSR:
10062 case VMX_EXIT_MWAIT:
10063 case VMX_EXIT_MONITOR:
10064 case VMX_EXIT_PAUSE:
10065 case VMX_EXIT_XDTR_ACCESS:
10066 case VMX_EXIT_TR_ACCESS:
10067 case VMX_EXIT_INVEPT:
10068 case VMX_EXIT_RDTSCP:
10069 case VMX_EXIT_INVVPID:
10070 case VMX_EXIT_WBINVD:
10071 case VMX_EXIT_XSETBV:
10072 case VMX_EXIT_RDRAND:
10073 case VMX_EXIT_INVPCID:
10074 case VMX_EXIT_VMFUNC:
10075 case VMX_EXIT_RDSEED:
10076 case VMX_EXIT_XSAVES:
10077 case VMX_EXIT_XRSTORS:
10078 {
10079 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10080 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10081 AssertRCReturn(rc2, rc2);
10082 if ( pMixedCtx->rip != pDbgState->uRipStart
10083 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10084 return VINF_EM_DBG_STEPPED;
10085 break;
10086 }
10087
10088 /* Errors and unexpected events: */
10089 case VMX_EXIT_INIT_SIGNAL:
10090 case VMX_EXIT_SIPI:
10091 case VMX_EXIT_IO_SMI:
10092 case VMX_EXIT_SMI:
10093 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10094 case VMX_EXIT_ERR_MSR_LOAD:
10095 case VMX_EXIT_ERR_MACHINE_CHECK:
10096 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10097 break;
10098
10099 default:
10100 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10101 break;
10102 }
10103 }
10104
10105 /*
10106 * Check for debugger event breakpoints and dtrace probes.
10107 */
10108 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10109 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10110 {
10111 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10112 if (rcStrict != VINF_SUCCESS)
10113 return rcStrict;
10114 }
10115
10116 /*
10117 * Normal processing.
10118 */
10119#ifdef HMVMX_USE_FUNCTION_TABLE
10120 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10121#else
10122 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10123#endif
10124}
10125
10126
10127/**
10128 * Single steps guest code using VT-x.
10129 *
10130 * @returns Strict VBox status code (i.e. informational status codes too).
10131 * @param pVM The cross context VM structure.
10132 * @param pVCpu The cross context virtual CPU structure.
10133 * @param pCtx Pointer to the guest-CPU context.
10134 *
10135 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10136 */
10137static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10138{
10139 VMXTRANSIENT VmxTransient;
10140 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10141
10142 /* Set HMCPU indicators. */
10143 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10144 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10145 pVCpu->hm.s.fDebugWantRdTscExit = false;
10146 pVCpu->hm.s.fUsingDebugLoop = true;
10147
10148 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10149 VMXRUNDBGSTATE DbgState;
10150 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10151 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10152
10153 /*
10154 * The loop.
10155 */
10156 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10157 for (uint32_t cLoops = 0; ; cLoops++)
10158 {
10159 Assert(!HMR0SuspendPending());
10160 HMVMX_ASSERT_CPU_SAFE();
10161 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10162
10163 /*
10164 * Preparatory work for running guest code, this may force us to return
10165 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10166 */
10167 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10168 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10169 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10170 if (rcStrict != VINF_SUCCESS)
10171 break;
10172
10173 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10174 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10175
10176 /*
10177 * Now we can run the guest code.
10178 */
10179 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10180
10181 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10182
10183 /*
10184 * Restore any residual host-state and save any bits shared between host
10185 * and guest into the guest-CPU state. Re-enables interrupts!
10186 */
10187 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10188
10189 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10190 if (RT_SUCCESS(rcRun))
10191 { /* very likely */ }
10192 else
10193 {
10194 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10195 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10196 return rcRun;
10197 }
10198
10199 /* Profile the VM-exit. */
10200 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10202 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10203 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10204 HMVMX_START_EXIT_DISPATCH_PROF();
10205
10206 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10207
10208 /*
10209 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10210 */
10211 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10212 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10213 if (rcStrict != VINF_SUCCESS)
10214 break;
10215 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10216 {
10217 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10218 rcStrict = VINF_EM_RAW_INTERRUPT;
10219 break;
10220 }
10221
10222 /*
10223 * Stepping: Did the RIP change, if so, consider it a single step.
10224 * Otherwise, make sure one of the TFs gets set.
10225 */
10226 if (fStepping)
10227 {
10228 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10229 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10230 AssertRCReturn(rc2, rc2);
10231 if ( pCtx->rip != DbgState.uRipStart
10232 || pCtx->cs.Sel != DbgState.uCsStart)
10233 {
10234 rcStrict = VINF_EM_DBG_STEPPED;
10235 break;
10236 }
10237 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10238 }
10239
10240 /*
10241 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10242 */
10243 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10244 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10245 }
10246
10247 /*
10248 * Clear the X86_EFL_TF if necessary.
10249 */
10250 if (pVCpu->hm.s.fClearTrapFlag)
10251 {
10252 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10253 AssertRCReturn(rc2, rc2);
10254 pVCpu->hm.s.fClearTrapFlag = false;
10255 pCtx->eflags.Bits.u1TF = 0;
10256 }
10257 /** @todo there seems to be issues with the resume flag when the monitor trap
10258 * flag is pending without being used. Seen early in bios init when
10259 * accessing APIC page in protected mode. */
10260
10261 /*
10262 * Restore VM-exit control settings as we may not reenter this function the
10263 * next time around.
10264 */
10265 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10266
10267 /* Restore HMCPU indicators. */
10268 pVCpu->hm.s.fUsingDebugLoop = false;
10269 pVCpu->hm.s.fDebugWantRdTscExit = false;
10270 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10271
10272 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10273 return rcStrict;
10274}
10275
10276
10277/** @} */
10278
10279
10280/**
10281 * Checks if any expensive dtrace probes are enabled and we should go to the
10282 * debug loop.
10283 *
10284 * @returns true if we should use debug loop, false if not.
10285 */
10286static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10287{
10288 /* It's probably faster to OR the raw 32-bit counter variables together.
10289 Since the variables are in an array and the probes are next to one
10290 another (more or less), we have good locality. So, better read
10291 eight-nine cache lines ever time and only have one conditional, than
10292 128+ conditionals, right? */
10293 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10294 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10295 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10296 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10297 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10298 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10299 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10300 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10301 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10302 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10303 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10304 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10305 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10306 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10307 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10308 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10309 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10310 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10311 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10312 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10313 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10314 ) != 0
10315 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10316 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10317 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10318 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10319 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10320 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10321 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10322 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10323 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10324 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10325 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10326 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10327 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10328 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10329 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10330 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10331 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10332 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10333 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10334 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10335 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10336 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10337 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10338 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10339 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10340 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10341 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10342 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10343 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10344 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10345 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10346 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10347 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10348 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10349 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10350 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10351 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10352 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10353 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10354 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10355 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10356 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10357 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10358 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10359 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10360 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10361 ) != 0
10362 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10363 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10364 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10365 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10366 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10367 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10368 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10369 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10370 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10371 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10372 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10373 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10374 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10375 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10376 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10377 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10378 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10379 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10380 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10381 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10382 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10383 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10384 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10385 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10386 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10387 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10388 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10389 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10390 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10391 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10392 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10393 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10394 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10395 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10396 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10397 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10398 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10399 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10400 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10401 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10402 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10403 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10404 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10405 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10406 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10407 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10408 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10409 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10410 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10411 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10412 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10413 ) != 0;
10414}
10415
10416
10417/**
10418 * Runs the guest code using VT-x.
10419 *
10420 * @returns Strict VBox status code (i.e. informational status codes too).
10421 * @param pVM The cross context VM structure.
10422 * @param pVCpu The cross context virtual CPU structure.
10423 * @param pCtx Pointer to the guest-CPU context.
10424 */
10425VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10426{
10427 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10428 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10429 HMVMX_ASSERT_PREEMPT_SAFE();
10430
10431 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10432
10433 VBOXSTRICTRC rcStrict;
10434 if ( !pVCpu->hm.s.fUseDebugLoop
10435 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10436 && !DBGFIsStepping(pVCpu)
10437 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10438 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10439 else
10440 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10441
10442 if (rcStrict == VERR_EM_INTERPRETER)
10443 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10444 else if (rcStrict == VINF_EM_RESET)
10445 rcStrict = VINF_EM_TRIPLE_FAULT;
10446
10447 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10448 if (RT_FAILURE(rc2))
10449 {
10450 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10451 rcStrict = rc2;
10452 }
10453 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10454 return rcStrict;
10455}
10456
10457
10458#ifndef HMVMX_USE_FUNCTION_TABLE
10459DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10460{
10461# ifdef DEBUG_ramshankar
10462# define RETURN_EXIT_CALL(a_CallExpr) \
10463 do { \
10464 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10465 VBOXSTRICTRC rcStrict = a_CallExpr; \
10466 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10467 return rcStrict; \
10468 } while (0)
10469# else
10470# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10471# endif
10472 switch (rcReason)
10473 {
10474 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10475 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10476 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10477 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10478 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10479 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10480 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10481 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10494 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10499 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10500 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10501 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10502 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10503 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10504 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10505 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10506 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10507 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10508
10509 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10510 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10511 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10512 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10513 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10514 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10515 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10516 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10517 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10518
10519 case VMX_EXIT_VMCLEAR:
10520 case VMX_EXIT_VMLAUNCH:
10521 case VMX_EXIT_VMPTRLD:
10522 case VMX_EXIT_VMPTRST:
10523 case VMX_EXIT_VMREAD:
10524 case VMX_EXIT_VMRESUME:
10525 case VMX_EXIT_VMWRITE:
10526 case VMX_EXIT_VMXOFF:
10527 case VMX_EXIT_VMXON:
10528 case VMX_EXIT_INVEPT:
10529 case VMX_EXIT_INVVPID:
10530 case VMX_EXIT_VMFUNC:
10531 case VMX_EXIT_XSAVES:
10532 case VMX_EXIT_XRSTORS:
10533 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10534 case VMX_EXIT_ENCLS:
10535 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10536 case VMX_EXIT_PML_FULL:
10537 default:
10538 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10539 }
10540#undef RETURN_EXIT_CALL
10541}
10542#endif /* !HMVMX_USE_FUNCTION_TABLE */
10543
10544
10545#ifdef VBOX_STRICT
10546/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10547# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10548 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10549
10550# define HMVMX_ASSERT_PREEMPT_CPUID() \
10551 do { \
10552 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10553 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10554 } while (0)
10555
10556# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10557 do { \
10558 AssertPtr(pVCpu); \
10559 AssertPtr(pMixedCtx); \
10560 AssertPtr(pVmxTransient); \
10561 Assert(pVmxTransient->fVMEntryFailed == false); \
10562 Assert(ASMIntAreEnabled()); \
10563 HMVMX_ASSERT_PREEMPT_SAFE(); \
10564 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10565 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10566 HMVMX_ASSERT_PREEMPT_SAFE(); \
10567 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10568 HMVMX_ASSERT_PREEMPT_CPUID(); \
10569 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10570 } while (0)
10571
10572# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10573 do { \
10574 Log4Func(("\n")); \
10575 } while (0)
10576#else /* nonstrict builds: */
10577# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10578 do { \
10579 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10580 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10581 } while (0)
10582# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10583#endif
10584
10585
10586/**
10587 * Advances the guest RIP by the specified number of bytes.
10588 *
10589 * @param pVCpu The cross context virtual CPU structure.
10590 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10591 * out-of-sync. Make sure to update the required fields
10592 * before using them.
10593 * @param cbInstr Number of bytes to advance the RIP by.
10594 *
10595 * @remarks No-long-jump zone!!!
10596 */
10597DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10598{
10599 /* Advance the RIP. */
10600 pMixedCtx->rip += cbInstr;
10601 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10602
10603 /* Update interrupt inhibition. */
10604 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10605 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10606 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10607}
10608
10609
10610/**
10611 * Advances the guest RIP after reading it from the VMCS.
10612 *
10613 * @returns VBox status code, no informational status codes.
10614 * @param pVCpu The cross context virtual CPU structure.
10615 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10616 * out-of-sync. Make sure to update the required fields
10617 * before using them.
10618 * @param pVmxTransient Pointer to the VMX transient structure.
10619 *
10620 * @remarks No-long-jump zone!!!
10621 */
10622static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10623{
10624 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10625 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10626 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10627 AssertRCReturn(rc, rc);
10628
10629 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10630
10631 /*
10632 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10633 * pending debug exception field as it takes care of priority of events.
10634 *
10635 * See Intel spec. 32.2.1 "Debug Exceptions".
10636 */
10637 if ( !pVCpu->hm.s.fSingleInstruction
10638 && pMixedCtx->eflags.Bits.u1TF)
10639 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10640
10641 return VINF_SUCCESS;
10642}
10643
10644
10645/**
10646 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10647 * and update error record fields accordingly.
10648 *
10649 * @return VMX_IGS_* return codes.
10650 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10651 * wrong with the guest state.
10652 *
10653 * @param pVM The cross context VM structure.
10654 * @param pVCpu The cross context virtual CPU structure.
10655 * @param pCtx Pointer to the guest-CPU state.
10656 *
10657 * @remarks This function assumes our cache of the VMCS controls
10658 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10659 */
10660static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10661{
10662#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10663#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10664 uError = (err); \
10665 break; \
10666 } else do { } while (0)
10667
10668 int rc;
10669 uint32_t uError = VMX_IGS_ERROR;
10670 uint32_t u32Val;
10671 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10672
10673 do
10674 {
10675 /*
10676 * CR0.
10677 */
10678 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10679 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10680 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10681 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10682 if (fUnrestrictedGuest)
10683 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10684
10685 uint32_t u32GuestCR0;
10686 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10687 AssertRCBreak(rc);
10688 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10689 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10690 if ( !fUnrestrictedGuest
10691 && (u32GuestCR0 & X86_CR0_PG)
10692 && !(u32GuestCR0 & X86_CR0_PE))
10693 {
10694 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10695 }
10696
10697 /*
10698 * CR4.
10699 */
10700 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10701 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10702
10703 uint32_t u32GuestCR4;
10704 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10705 AssertRCBreak(rc);
10706 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10707 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10708
10709 /*
10710 * IA32_DEBUGCTL MSR.
10711 */
10712 uint64_t u64Val;
10713 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10714 AssertRCBreak(rc);
10715 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10716 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10717 {
10718 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10719 }
10720 uint64_t u64DebugCtlMsr = u64Val;
10721
10722#ifdef VBOX_STRICT
10723 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10724 AssertRCBreak(rc);
10725 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10726#endif
10727 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10728
10729 /*
10730 * RIP and RFLAGS.
10731 */
10732 uint32_t u32Eflags;
10733#if HC_ARCH_BITS == 64
10734 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10735 AssertRCBreak(rc);
10736 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10737 if ( !fLongModeGuest
10738 || !pCtx->cs.Attr.n.u1Long)
10739 {
10740 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10741 }
10742 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10743 * must be identical if the "IA-32e mode guest" VM-entry
10744 * control is 1 and CS.L is 1. No check applies if the
10745 * CPU supports 64 linear-address bits. */
10746
10747 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10748 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10749 AssertRCBreak(rc);
10750 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10751 VMX_IGS_RFLAGS_RESERVED);
10752 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10753 u32Eflags = u64Val;
10754#else
10755 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10756 AssertRCBreak(rc);
10757 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10758 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10759#endif
10760
10761 if ( fLongModeGuest
10762 || ( fUnrestrictedGuest
10763 && !(u32GuestCR0 & X86_CR0_PE)))
10764 {
10765 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10766 }
10767
10768 uint32_t u32EntryInfo;
10769 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10770 AssertRCBreak(rc);
10771 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10772 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10773 {
10774 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10775 }
10776
10777 /*
10778 * 64-bit checks.
10779 */
10780#if HC_ARCH_BITS == 64
10781 if (fLongModeGuest)
10782 {
10783 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10784 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10785 }
10786
10787 if ( !fLongModeGuest
10788 && (u32GuestCR4 & X86_CR4_PCIDE))
10789 {
10790 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10791 }
10792
10793 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10794 * 51:32 beyond the processor's physical-address width are 0. */
10795
10796 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10797 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10798 {
10799 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10800 }
10801
10802 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10803 AssertRCBreak(rc);
10804 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10805
10806 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10807 AssertRCBreak(rc);
10808 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10809#endif
10810
10811 /*
10812 * PERF_GLOBAL MSR.
10813 */
10814 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10815 {
10816 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10817 AssertRCBreak(rc);
10818 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10819 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10820 }
10821
10822 /*
10823 * PAT MSR.
10824 */
10825 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10826 {
10827 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10828 AssertRCBreak(rc);
10829 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10830 for (unsigned i = 0; i < 8; i++)
10831 {
10832 uint8_t u8Val = (u64Val & 0xff);
10833 if ( u8Val != 0 /* UC */
10834 && u8Val != 1 /* WC */
10835 && u8Val != 4 /* WT */
10836 && u8Val != 5 /* WP */
10837 && u8Val != 6 /* WB */
10838 && u8Val != 7 /* UC- */)
10839 {
10840 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10841 }
10842 u64Val >>= 8;
10843 }
10844 }
10845
10846 /*
10847 * EFER MSR.
10848 */
10849 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10850 {
10851 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10852 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10853 AssertRCBreak(rc);
10854 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10855 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10856 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10857 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10858 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10859 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10860 || !(u32GuestCR0 & X86_CR0_PG)
10861 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10862 VMX_IGS_EFER_LMA_LME_MISMATCH);
10863 }
10864
10865 /*
10866 * Segment registers.
10867 */
10868 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10869 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10870 if (!(u32Eflags & X86_EFL_VM))
10871 {
10872 /* CS */
10873 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10874 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10875 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10876 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10877 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10878 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10879 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10880 /* CS cannot be loaded with NULL in protected mode. */
10881 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10882 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10883 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10884 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10885 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10886 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10887 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10888 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10889 else
10890 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10891
10892 /* SS */
10893 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10894 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10895 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10896 if ( !(pCtx->cr0 & X86_CR0_PE)
10897 || pCtx->cs.Attr.n.u4Type == 3)
10898 {
10899 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10900 }
10901 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10902 {
10903 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10904 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10905 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10906 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10907 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10908 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10909 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10910 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10911 }
10912
10913 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10914 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10915 {
10916 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10917 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10918 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10919 || pCtx->ds.Attr.n.u4Type > 11
10920 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10921 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10922 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10923 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10924 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10925 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10926 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10927 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10928 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10929 }
10930 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10931 {
10932 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10933 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10934 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10935 || pCtx->es.Attr.n.u4Type > 11
10936 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10937 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10938 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10939 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10940 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10941 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10942 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10943 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10944 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10945 }
10946 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10947 {
10948 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10949 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10950 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10951 || pCtx->fs.Attr.n.u4Type > 11
10952 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10953 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10954 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10955 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10956 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10957 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10958 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10959 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10960 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10961 }
10962 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10963 {
10964 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10965 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10966 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10967 || pCtx->gs.Attr.n.u4Type > 11
10968 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10969 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10970 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10971 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10972 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10973 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10974 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10975 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10976 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10977 }
10978 /* 64-bit capable CPUs. */
10979#if HC_ARCH_BITS == 64
10980 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10981 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10982 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10983 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10984 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10985 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10986 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10987 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10988 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10989 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10990 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10991#endif
10992 }
10993 else
10994 {
10995 /* V86 mode checks. */
10996 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10997 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10998 {
10999 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11000 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11001 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11002 }
11003 else
11004 {
11005 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11006 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11007 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11008 }
11009
11010 /* CS */
11011 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11012 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11013 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11014 /* SS */
11015 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11016 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11017 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11018 /* DS */
11019 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11020 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11021 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11022 /* ES */
11023 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11024 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11025 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11026 /* FS */
11027 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11028 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11029 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11030 /* GS */
11031 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11032 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11033 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11034 /* 64-bit capable CPUs. */
11035#if HC_ARCH_BITS == 64
11036 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11037 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11038 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11039 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11040 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11041 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11042 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11043 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11044 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11045 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11046 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11047#endif
11048 }
11049
11050 /*
11051 * TR.
11052 */
11053 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11054 /* 64-bit capable CPUs. */
11055#if HC_ARCH_BITS == 64
11056 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11057#endif
11058 if (fLongModeGuest)
11059 {
11060 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11061 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11062 }
11063 else
11064 {
11065 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11066 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11067 VMX_IGS_TR_ATTR_TYPE_INVALID);
11068 }
11069 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11070 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11071 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11072 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11073 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11074 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11075 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11076 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11077
11078 /*
11079 * GDTR and IDTR.
11080 */
11081#if HC_ARCH_BITS == 64
11082 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11083 AssertRCBreak(rc);
11084 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11085
11086 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11087 AssertRCBreak(rc);
11088 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11089#endif
11090
11091 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11092 AssertRCBreak(rc);
11093 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11094
11095 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11096 AssertRCBreak(rc);
11097 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11098
11099 /*
11100 * Guest Non-Register State.
11101 */
11102 /* Activity State. */
11103 uint32_t u32ActivityState;
11104 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11105 AssertRCBreak(rc);
11106 HMVMX_CHECK_BREAK( !u32ActivityState
11107 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11108 VMX_IGS_ACTIVITY_STATE_INVALID);
11109 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11110 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11111 uint32_t u32IntrState;
11112 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11113 AssertRCBreak(rc);
11114 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11115 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11116 {
11117 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11118 }
11119
11120 /** @todo Activity state and injecting interrupts. Left as a todo since we
11121 * currently don't use activity states but ACTIVE. */
11122
11123 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11124 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11125
11126 /* Guest interruptibility-state. */
11127 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11128 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11129 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11130 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11131 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11132 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11133 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11134 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11135 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11136 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11137 {
11138 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11139 {
11140 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11141 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11142 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11143 }
11144 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11145 {
11146 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11147 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11148 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11149 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11150 }
11151 }
11152 /** @todo Assumes the processor is not in SMM. */
11153 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11154 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11155 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11156 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11157 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11158 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11159 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11160 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11161 {
11162 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11163 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11164 }
11165
11166 /* Pending debug exceptions. */
11167#if HC_ARCH_BITS == 64
11168 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11169 AssertRCBreak(rc);
11170 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11171 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11172 u32Val = u64Val; /* For pending debug exceptions checks below. */
11173#else
11174 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11175 AssertRCBreak(rc);
11176 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11177 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11178#endif
11179
11180 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11181 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11182 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11183 {
11184 if ( (u32Eflags & X86_EFL_TF)
11185 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11186 {
11187 /* Bit 14 is PendingDebug.BS. */
11188 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11189 }
11190 if ( !(u32Eflags & X86_EFL_TF)
11191 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11192 {
11193 /* Bit 14 is PendingDebug.BS. */
11194 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11195 }
11196 }
11197
11198 /* VMCS link pointer. */
11199 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11200 AssertRCBreak(rc);
11201 if (u64Val != UINT64_C(0xffffffffffffffff))
11202 {
11203 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11204 /** @todo Bits beyond the processor's physical-address width MBZ. */
11205 /** @todo 32-bit located in memory referenced by value of this field (as a
11206 * physical address) must contain the processor's VMCS revision ID. */
11207 /** @todo SMM checks. */
11208 }
11209
11210 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11211 * not using Nested Paging? */
11212 if ( pVM->hm.s.fNestedPaging
11213 && !fLongModeGuest
11214 && CPUMIsGuestInPAEModeEx(pCtx))
11215 {
11216 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11217 AssertRCBreak(rc);
11218 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11219
11220 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11221 AssertRCBreak(rc);
11222 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11223
11224 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11225 AssertRCBreak(rc);
11226 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11227
11228 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11229 AssertRCBreak(rc);
11230 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11231 }
11232
11233 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11234 if (uError == VMX_IGS_ERROR)
11235 uError = VMX_IGS_REASON_NOT_FOUND;
11236 } while (0);
11237
11238 pVCpu->hm.s.u32HMError = uError;
11239 return uError;
11240
11241#undef HMVMX_ERROR_BREAK
11242#undef HMVMX_CHECK_BREAK
11243}
11244
11245/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11246/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11247/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11248
11249/** @name VM-exit handlers.
11250 * @{
11251 */
11252
11253/**
11254 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11255 */
11256HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257{
11258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11260 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11261 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11262 return VINF_SUCCESS;
11263 return VINF_EM_RAW_INTERRUPT;
11264}
11265
11266
11267/**
11268 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11269 */
11270HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11271{
11272 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11273 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11274
11275 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11276 AssertRCReturn(rc, rc);
11277
11278 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11279 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11280 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11281 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11282
11283 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11284 {
11285 /*
11286 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11287 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11288 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11289 *
11290 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11291 */
11292 VMXDispatchHostNmi();
11293 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11294 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11295 return VINF_SUCCESS;
11296 }
11297
11298 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11299 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11300 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11301 { /* likely */ }
11302 else
11303 {
11304 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11305 rcStrictRc1 = VINF_SUCCESS;
11306 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11307 return rcStrictRc1;
11308 }
11309
11310 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11311 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11312 switch (uIntType)
11313 {
11314 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11315 Assert(uVector == X86_XCPT_DB);
11316 /* no break */
11317 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11318 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11319 /* no break */
11320 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11321 {
11322 /*
11323 * If there's any exception caused as a result of event injection, go back to
11324 * the interpreter. The page-fault case is complicated and we manually handle
11325 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11326 * handled in hmR0VmxCheckExitDueToEventDelivery.
11327 */
11328 if (!pVCpu->hm.s.Event.fPending)
11329 { /* likely */ }
11330 else if ( uVector != X86_XCPT_PF
11331 && uVector != X86_XCPT_AC)
11332 {
11333 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11334 rc = VERR_EM_INTERPRETER;
11335 break;
11336 }
11337
11338 switch (uVector)
11339 {
11340 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11341 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11342 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11343 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11344 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11345 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11346 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11347
11348 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11349 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11350 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11351 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11352 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11353 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11354 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11355 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11356 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11357 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11358 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11359 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11360 default:
11361 {
11362 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11363 AssertRCReturn(rc, rc);
11364
11365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11366 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11367 {
11368 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11369 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11370 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11371
11372 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11373 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11374 AssertRCReturn(rc, rc);
11375 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11376 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11377 0 /* GCPtrFaultAddress */);
11378 AssertRCReturn(rc, rc);
11379 }
11380 else
11381 {
11382 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11383 pVCpu->hm.s.u32HMError = uVector;
11384 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11385 }
11386 break;
11387 }
11388 }
11389 break;
11390 }
11391
11392 default:
11393 {
11394 pVCpu->hm.s.u32HMError = uExitIntInfo;
11395 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11396 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11397 break;
11398 }
11399 }
11400 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11401 return rc;
11402}
11403
11404
11405/**
11406 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11407 */
11408HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11409{
11410 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11411
11412 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11413 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11414
11415 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11416 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11417 return VINF_SUCCESS;
11418}
11419
11420
11421/**
11422 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11423 */
11424HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11425{
11426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11427 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11428 {
11429 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11430 HMVMX_RETURN_UNEXPECTED_EXIT();
11431 }
11432
11433 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11434
11435 /*
11436 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11437 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11438 */
11439 uint32_t uIntrState = 0;
11440 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11441 AssertRCReturn(rc, rc);
11442
11443 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11444 if ( fBlockSti
11445 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11446 {
11447 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11448 }
11449
11450 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11451 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11452
11453 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11454 return VINF_SUCCESS;
11455}
11456
11457
11458/**
11459 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11460 */
11461HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11462{
11463 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11464 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11465 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11466}
11467
11468
11469/**
11470 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11471 */
11472HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11473{
11474 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11476 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11477}
11478
11479
11480/**
11481 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11482 */
11483HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11484{
11485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11486 PVM pVM = pVCpu->CTX_SUFF(pVM);
11487 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11488 if (RT_LIKELY(rc == VINF_SUCCESS))
11489 {
11490 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11491 Assert(pVmxTransient->cbInstr == 2);
11492 }
11493 else
11494 {
11495 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11496 rc = VERR_EM_INTERPRETER;
11497 }
11498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11499 return rc;
11500}
11501
11502
11503/**
11504 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11505 */
11506HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11507{
11508 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11509 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11510 AssertRCReturn(rc, rc);
11511
11512 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11513 return VINF_EM_RAW_EMULATE_INSTR;
11514
11515 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11516 HMVMX_RETURN_UNEXPECTED_EXIT();
11517}
11518
11519
11520/**
11521 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11522 */
11523HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11524{
11525 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11526 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11527 AssertRCReturn(rc, rc);
11528
11529 PVM pVM = pVCpu->CTX_SUFF(pVM);
11530 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11531 if (RT_LIKELY(rc == VINF_SUCCESS))
11532 {
11533 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11534 Assert(pVmxTransient->cbInstr == 2);
11535 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11536 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11537 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11538 }
11539 else
11540 rc = VERR_EM_INTERPRETER;
11541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11542 return rc;
11543}
11544
11545
11546/**
11547 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11548 */
11549HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11550{
11551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11552 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11553 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11554 AssertRCReturn(rc, rc);
11555
11556 PVM pVM = pVCpu->CTX_SUFF(pVM);
11557 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11558 if (RT_SUCCESS(rc))
11559 {
11560 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11561 Assert(pVmxTransient->cbInstr == 3);
11562 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11563 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11564 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11565 }
11566 else
11567 {
11568 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11569 rc = VERR_EM_INTERPRETER;
11570 }
11571 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11572 return rc;
11573}
11574
11575
11576/**
11577 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11578 */
11579HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11580{
11581 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11582 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11583 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11584 AssertRCReturn(rc, rc);
11585
11586 PVM pVM = pVCpu->CTX_SUFF(pVM);
11587 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11588 if (RT_LIKELY(rc == VINF_SUCCESS))
11589 {
11590 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11591 Assert(pVmxTransient->cbInstr == 2);
11592 }
11593 else
11594 {
11595 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11596 rc = VERR_EM_INTERPRETER;
11597 }
11598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11599 return rc;
11600}
11601
11602
11603/**
11604 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11605 */
11606HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11607{
11608 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11609 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11610
11611 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11612 if (pVCpu->hm.s.fHypercallsEnabled)
11613 {
11614#if 0
11615 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11616#else
11617 /* Aggressive state sync. for now. */
11618 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11619 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11620 AssertRCReturn(rc, rc);
11621#endif
11622
11623 /* Perform the hypercall. */
11624 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11625 if (rcStrict == VINF_SUCCESS)
11626 {
11627 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11628 AssertRCReturn(rc, rc);
11629 }
11630 else
11631 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11632 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11633 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11634
11635 /* If the hypercall changes anything other than guest's general-purpose registers,
11636 we would need to reload the guest changed bits here before VM-entry. */
11637 }
11638 else
11639 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11640
11641 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11642 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11643 {
11644 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11645 rcStrict = VINF_SUCCESS;
11646 }
11647
11648 return rcStrict;
11649}
11650
11651
11652/**
11653 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11654 */
11655HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11656{
11657 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11658 PVM pVM = pVCpu->CTX_SUFF(pVM);
11659 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11660
11661 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11662 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11663 AssertRCReturn(rc, rc);
11664
11665 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11666 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11667 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11668 else
11669 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11670 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11672 return rcStrict;
11673}
11674
11675
11676/**
11677 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11678 */
11679HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11680{
11681 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11682 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11683 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11684 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11685 AssertRCReturn(rc, rc);
11686
11687 PVM pVM = pVCpu->CTX_SUFF(pVM);
11688 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11689 if (RT_LIKELY(rc == VINF_SUCCESS))
11690 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11691 else
11692 {
11693 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11694 rc = VERR_EM_INTERPRETER;
11695 }
11696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11697 return rc;
11698}
11699
11700
11701/**
11702 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11703 */
11704HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11705{
11706 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11707 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11708 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11709 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11710 AssertRCReturn(rc, rc);
11711
11712 PVM pVM = pVCpu->CTX_SUFF(pVM);
11713 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11714 rc = VBOXSTRICTRC_VAL(rc2);
11715 if (RT_LIKELY( rc == VINF_SUCCESS
11716 || rc == VINF_EM_HALT))
11717 {
11718 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11719 AssertRCReturn(rc3, rc3);
11720
11721 if ( rc == VINF_EM_HALT
11722 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11723 {
11724 rc = VINF_SUCCESS;
11725 }
11726 }
11727 else
11728 {
11729 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11730 rc = VERR_EM_INTERPRETER;
11731 }
11732 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11733 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11734 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11735 return rc;
11736}
11737
11738
11739/**
11740 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11741 */
11742HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11743{
11744 /*
11745 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11746 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11747 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11748 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11749 */
11750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11751 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11752 HMVMX_RETURN_UNEXPECTED_EXIT();
11753}
11754
11755
11756/**
11757 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11758 */
11759HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11760{
11761 /*
11762 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11763 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11764 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11765 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11766 */
11767 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11768 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11769 HMVMX_RETURN_UNEXPECTED_EXIT();
11770}
11771
11772
11773/**
11774 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11775 */
11776HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11777{
11778 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11780 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11781 HMVMX_RETURN_UNEXPECTED_EXIT();
11782}
11783
11784
11785/**
11786 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11787 */
11788HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11789{
11790 /*
11791 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11792 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11793 * See Intel spec. 25.3 "Other Causes of VM-exits".
11794 */
11795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11796 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11797 HMVMX_RETURN_UNEXPECTED_EXIT();
11798}
11799
11800
11801/**
11802 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11803 * VM-exit.
11804 */
11805HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11806{
11807 /*
11808 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11809 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11810 *
11811 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11812 * See Intel spec. "23.8 Restrictions on VMX operation".
11813 */
11814 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11815 return VINF_SUCCESS;
11816}
11817
11818
11819/**
11820 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11821 * VM-exit.
11822 */
11823HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11824{
11825 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11826 return VINF_EM_RESET;
11827}
11828
11829
11830/**
11831 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11832 */
11833HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11834{
11835 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11836 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11837
11838 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11839 AssertRCReturn(rc, rc);
11840
11841 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11842 rc = VINF_SUCCESS;
11843 else
11844 rc = VINF_EM_HALT;
11845
11846 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11847 if (rc != VINF_SUCCESS)
11848 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11849 return rc;
11850}
11851
11852
11853/**
11854 * VM-exit handler for instructions that result in a \#UD exception delivered to
11855 * the guest.
11856 */
11857HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11858{
11859 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11860 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11861 return VINF_SUCCESS;
11862}
11863
11864
11865/**
11866 * VM-exit handler for expiry of the VMX preemption timer.
11867 */
11868HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11869{
11870 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11871
11872 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11873 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11874
11875 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11876 PVM pVM = pVCpu->CTX_SUFF(pVM);
11877 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11879 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11880}
11881
11882
11883/**
11884 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11885 */
11886HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11887{
11888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11889
11890 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11891 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11892 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11893 AssertRCReturn(rc, rc);
11894
11895 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11896 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11897
11898 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11899
11900 return rcStrict;
11901}
11902
11903
11904/**
11905 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11906 */
11907HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11908{
11909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11910
11911 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11912 /** @todo implement EMInterpretInvpcid() */
11913 return VERR_EM_INTERPRETER;
11914}
11915
11916
11917/**
11918 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11919 * Error VM-exit.
11920 */
11921HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11922{
11923 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11924 AssertRCReturn(rc, rc);
11925
11926 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11927 AssertRCReturn(rc, rc);
11928
11929 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11930 NOREF(uInvalidReason);
11931
11932#ifdef VBOX_STRICT
11933 uint32_t uIntrState;
11934 RTHCUINTREG uHCReg;
11935 uint64_t u64Val;
11936 uint32_t u32Val;
11937
11938 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11939 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11940 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11941 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11942 AssertRCReturn(rc, rc);
11943
11944 Log4(("uInvalidReason %u\n", uInvalidReason));
11945 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11946 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11947 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11948 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11949
11950 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11951 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11952 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11953 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11954 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11955 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11956 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11957 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11958 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11959 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11960 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11961 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11962#else
11963 NOREF(pVmxTransient);
11964#endif
11965
11966 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11967 return VERR_VMX_INVALID_GUEST_STATE;
11968}
11969
11970
11971/**
11972 * VM-exit handler for VM-entry failure due to an MSR-load
11973 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11974 */
11975HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11976{
11977 NOREF(pVmxTransient);
11978 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11979 HMVMX_RETURN_UNEXPECTED_EXIT();
11980}
11981
11982
11983/**
11984 * VM-exit handler for VM-entry failure due to a machine-check event
11985 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11986 */
11987HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11988{
11989 NOREF(pVmxTransient);
11990 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11991 HMVMX_RETURN_UNEXPECTED_EXIT();
11992}
11993
11994
11995/**
11996 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11997 * theory.
11998 */
11999HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12000{
12001 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12002 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12003 return VERR_VMX_UNDEFINED_EXIT_CODE;
12004}
12005
12006
12007/**
12008 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12009 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12010 * Conditional VM-exit.
12011 */
12012HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12013{
12014 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12015
12016 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12018 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12019 return VERR_EM_INTERPRETER;
12020 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12021 HMVMX_RETURN_UNEXPECTED_EXIT();
12022}
12023
12024
12025/**
12026 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12027 */
12028HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12029{
12030 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12031
12032 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12034 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12035 return VERR_EM_INTERPRETER;
12036 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12037 HMVMX_RETURN_UNEXPECTED_EXIT();
12038}
12039
12040
12041/**
12042 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12043 */
12044HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12045{
12046 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12047
12048 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12049 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12050 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12051 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12052 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12053 {
12054 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12055 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12056 }
12057 AssertRCReturn(rc, rc);
12058 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12059
12060#ifdef VBOX_STRICT
12061 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12062 {
12063 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12064 && pMixedCtx->ecx != MSR_K6_EFER)
12065 {
12066 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12067 pMixedCtx->ecx));
12068 HMVMX_RETURN_UNEXPECTED_EXIT();
12069 }
12070 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12071 {
12072 VMXMSREXITREAD enmRead;
12073 VMXMSREXITWRITE enmWrite;
12074 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12075 AssertRCReturn(rc2, rc2);
12076 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12077 {
12078 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12079 HMVMX_RETURN_UNEXPECTED_EXIT();
12080 }
12081 }
12082 }
12083#endif
12084
12085 PVM pVM = pVCpu->CTX_SUFF(pVM);
12086 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12087 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12088 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12089 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12090 if (RT_SUCCESS(rc))
12091 {
12092 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12093 Assert(pVmxTransient->cbInstr == 2);
12094 }
12095 return rc;
12096}
12097
12098
12099/**
12100 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12101 */
12102HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12103{
12104 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12105 PVM pVM = pVCpu->CTX_SUFF(pVM);
12106 int rc = VINF_SUCCESS;
12107
12108 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12109 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12110 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12111 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12112 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12113 {
12114 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12115 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12116 }
12117 AssertRCReturn(rc, rc);
12118 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12119
12120 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12121 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12123
12124 if (RT_SUCCESS(rc))
12125 {
12126 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12127
12128 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12129 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12130 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12131 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12132 {
12133 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12134 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12135 EMInterpretWrmsr() changes it. */
12136 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12137 }
12138 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12139 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12140 else if (pMixedCtx->ecx == MSR_K6_EFER)
12141 {
12142 /*
12143 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12144 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12145 * the other bits as well, SCE and NXE. See @bugref{7368}.
12146 */
12147 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12148 }
12149
12150 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12151 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12152 {
12153 switch (pMixedCtx->ecx)
12154 {
12155 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12156 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12157 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12158 case MSR_K8_FS_BASE: /* no break */
12159 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12160 case MSR_K6_EFER: /* already handled above */ break;
12161 default:
12162 {
12163 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12164 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12165 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12166 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12167 break;
12168 }
12169 }
12170 }
12171#ifdef VBOX_STRICT
12172 else
12173 {
12174 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12175 switch (pMixedCtx->ecx)
12176 {
12177 case MSR_IA32_SYSENTER_CS:
12178 case MSR_IA32_SYSENTER_EIP:
12179 case MSR_IA32_SYSENTER_ESP:
12180 case MSR_K8_FS_BASE:
12181 case MSR_K8_GS_BASE:
12182 {
12183 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12184 HMVMX_RETURN_UNEXPECTED_EXIT();
12185 }
12186
12187 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12188 default:
12189 {
12190 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12191 {
12192 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12193 if (pMixedCtx->ecx != MSR_K6_EFER)
12194 {
12195 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12196 pMixedCtx->ecx));
12197 HMVMX_RETURN_UNEXPECTED_EXIT();
12198 }
12199 }
12200
12201 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12202 {
12203 VMXMSREXITREAD enmRead;
12204 VMXMSREXITWRITE enmWrite;
12205 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12206 AssertRCReturn(rc2, rc2);
12207 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12208 {
12209 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12210 HMVMX_RETURN_UNEXPECTED_EXIT();
12211 }
12212 }
12213 break;
12214 }
12215 }
12216 }
12217#endif /* VBOX_STRICT */
12218 }
12219 return rc;
12220}
12221
12222
12223/**
12224 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12225 */
12226HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12227{
12228 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12229
12230 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12231 return VINF_EM_RAW_INTERRUPT;
12232}
12233
12234
12235/**
12236 * VM-exit handler for when the TPR value is lowered below the specified
12237 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12238 */
12239HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12240{
12241 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12242 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12243
12244 /*
12245 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12246 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12247 */
12248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12249 return VINF_SUCCESS;
12250}
12251
12252
12253/**
12254 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12255 * VM-exit.
12256 *
12257 * @retval VINF_SUCCESS when guest execution can continue.
12258 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12259 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12260 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12261 * interpreter.
12262 */
12263HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12264{
12265 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12266 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12267 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12268 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12269 AssertRCReturn(rc, rc);
12270
12271 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12272 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12273 PVM pVM = pVCpu->CTX_SUFF(pVM);
12274 VBOXSTRICTRC rcStrict;
12275 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12276 switch (uAccessType)
12277 {
12278 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12279 {
12280 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12281 AssertRCReturn(rc, rc);
12282
12283 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12284 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12285 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12286 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12287 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12288 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12289 {
12290 case 0: /* CR0 */
12291 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12292 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12293 break;
12294 case 2: /* CR2 */
12295 /* Nothing to do here, CR2 it's not part of the VMCS. */
12296 break;
12297 case 3: /* CR3 */
12298 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12299 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12300 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12301 break;
12302 case 4: /* CR4 */
12303 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12304 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12305 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12306 break;
12307 case 8: /* CR8 */
12308 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12309 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12310 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12311 break;
12312 default:
12313 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12314 break;
12315 }
12316
12317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12318 break;
12319 }
12320
12321 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12322 {
12323 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12324 AssertRCReturn(rc, rc);
12325
12326 Assert( !pVM->hm.s.fNestedPaging
12327 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12328 || pVCpu->hm.s.fUsingDebugLoop
12329 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12330
12331 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12332 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12333 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12334
12335 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12336 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12337 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12338 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12340 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12341 VBOXSTRICTRC_VAL(rcStrict)));
12342 break;
12343 }
12344
12345 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12346 {
12347 AssertRCReturn(rc, rc);
12348 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12349 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12350 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12352 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12353 break;
12354 }
12355
12356 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12357 {
12358 AssertRCReturn(rc, rc);
12359 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12360 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12361 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12362 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12364 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12365 break;
12366 }
12367
12368 default:
12369 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12370 VERR_VMX_UNEXPECTED_EXCEPTION);
12371 }
12372
12373 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12374 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12375 NOREF(pVM);
12376 return rcStrict;
12377}
12378
12379
12380/**
12381 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12382 * VM-exit.
12383 */
12384HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12385{
12386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12387 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12388
12389 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12390 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12391 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12392 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12393 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12394 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12395 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12396 AssertRCReturn(rc2, rc2);
12397
12398 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12399 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12400 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12401 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12402 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12403 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12404 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12405 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12406 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12407
12408 /* I/O operation lookup arrays. */
12409 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12410 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12411
12412 VBOXSTRICTRC rcStrict;
12413 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12414 uint32_t const cbInstr = pVmxTransient->cbInstr;
12415 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12416 PVM pVM = pVCpu->CTX_SUFF(pVM);
12417 if (fIOString)
12418 {
12419#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12420 See @bugref{5752#c158}. Should work now. */
12421 /*
12422 * INS/OUTS - I/O String instruction.
12423 *
12424 * Use instruction-information if available, otherwise fall back on
12425 * interpreting the instruction.
12426 */
12427 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12428 fIOWrite ? 'w' : 'r'));
12429 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12430 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12431 {
12432 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12433 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12434 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12435 AssertRCReturn(rc2, rc2);
12436 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12437 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12438 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12439 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12440 if (fIOWrite)
12441 {
12442 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12443 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12444 }
12445 else
12446 {
12447 /*
12448 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12449 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12450 * See Intel Instruction spec. for "INS".
12451 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12452 */
12453 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12454 }
12455 }
12456 else
12457 {
12458 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12459 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12460 AssertRCReturn(rc2, rc2);
12461 rcStrict = IEMExecOne(pVCpu);
12462 }
12463 /** @todo IEM needs to be setting these flags somehow. */
12464 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12465 fUpdateRipAlready = true;
12466#else
12467 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12468 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12469 if (RT_SUCCESS(rcStrict))
12470 {
12471 if (fIOWrite)
12472 {
12473 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12474 (DISCPUMODE)pDis->uAddrMode, cbValue);
12475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12476 }
12477 else
12478 {
12479 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12480 (DISCPUMODE)pDis->uAddrMode, cbValue);
12481 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12482 }
12483 }
12484 else
12485 {
12486 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12487 pMixedCtx->rip));
12488 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12489 }
12490#endif
12491 }
12492 else
12493 {
12494 /*
12495 * IN/OUT - I/O instruction.
12496 */
12497 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12498 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12499 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12500 if (fIOWrite)
12501 {
12502 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12504 }
12505 else
12506 {
12507 uint32_t u32Result = 0;
12508 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12509 if (IOM_SUCCESS(rcStrict))
12510 {
12511 /* Save result of I/O IN instr. in AL/AX/EAX. */
12512 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12513 }
12514 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12515 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12517 }
12518 }
12519
12520 if (IOM_SUCCESS(rcStrict))
12521 {
12522 if (!fUpdateRipAlready)
12523 {
12524 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12525 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12526 }
12527
12528 /*
12529 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12530 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12531 */
12532 if (fIOString)
12533 {
12534 /** @todo Single-step for INS/OUTS with REP prefix? */
12535 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12536 }
12537 else if ( !fDbgStepping
12538 && fGstStepping)
12539 {
12540 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12541 }
12542
12543 /*
12544 * If any I/O breakpoints are armed, we need to check if one triggered
12545 * and take appropriate action.
12546 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12547 */
12548 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12549 AssertRCReturn(rc2, rc2);
12550
12551 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12552 * execution engines about whether hyper BPs and such are pending. */
12553 uint32_t const uDr7 = pMixedCtx->dr[7];
12554 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12555 && X86_DR7_ANY_RW_IO(uDr7)
12556 && (pMixedCtx->cr4 & X86_CR4_DE))
12557 || DBGFBpIsHwIoArmed(pVM)))
12558 {
12559 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12560
12561 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12562 VMMRZCallRing3Disable(pVCpu);
12563 HM_DISABLE_PREEMPT();
12564
12565 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12566
12567 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12568 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12569 {
12570 /* Raise #DB. */
12571 if (fIsGuestDbgActive)
12572 ASMSetDR6(pMixedCtx->dr[6]);
12573 if (pMixedCtx->dr[7] != uDr7)
12574 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12575
12576 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12577 }
12578 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12579 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12580 else if ( rcStrict2 != VINF_SUCCESS
12581 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12582 rcStrict = rcStrict2;
12583 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12584
12585 HM_RESTORE_PREEMPT();
12586 VMMRZCallRing3Enable(pVCpu);
12587 }
12588 }
12589
12590#ifdef VBOX_STRICT
12591 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12592 Assert(!fIOWrite);
12593 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12594 Assert(fIOWrite);
12595 else
12596 {
12597#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12598 * statuses, that the VMM device and some others may return. See
12599 * IOM_SUCCESS() for guidance. */
12600 AssertMsg( RT_FAILURE(rcStrict)
12601 || rcStrict == VINF_SUCCESS
12602 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12603 || rcStrict == VINF_EM_DBG_BREAKPOINT
12604 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12605 || rcStrict == VINF_EM_RAW_TO_R3
12606 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12607#endif
12608 }
12609#endif
12610
12611 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12612 return rcStrict;
12613}
12614
12615
12616/**
12617 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12618 * VM-exit.
12619 */
12620HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12621{
12622 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12623
12624 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12625 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12626 AssertRCReturn(rc, rc);
12627 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12628 {
12629 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12630 AssertRCReturn(rc, rc);
12631 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12632 {
12633 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12634
12635 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12636 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12637
12638 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12639 Assert(!pVCpu->hm.s.Event.fPending);
12640 pVCpu->hm.s.Event.fPending = true;
12641 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12642 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12643 AssertRCReturn(rc, rc);
12644 if (fErrorCodeValid)
12645 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12646 else
12647 pVCpu->hm.s.Event.u32ErrCode = 0;
12648 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12649 && uVector == X86_XCPT_PF)
12650 {
12651 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12652 }
12653
12654 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12656 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12657 }
12658 }
12659
12660 /* Fall back to the interpreter to emulate the task-switch. */
12661 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12662 return VERR_EM_INTERPRETER;
12663}
12664
12665
12666/**
12667 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12668 */
12669HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12670{
12671 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12672 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12673 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12674 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12675 AssertRCReturn(rc, rc);
12676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12677 return VINF_EM_DBG_STEPPED;
12678}
12679
12680
12681/**
12682 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12683 */
12684HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12685{
12686 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12687
12688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12689
12690 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12691 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12692 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12693 {
12694 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12695 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12696 {
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12698 return VERR_EM_INTERPRETER;
12699 }
12700 }
12701 else
12702 {
12703 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12704 rcStrict1 = VINF_SUCCESS;
12705 return rcStrict1;
12706 }
12707
12708#if 0
12709 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12710 * just sync the whole thing. */
12711 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12712#else
12713 /* Aggressive state sync. for now. */
12714 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12715 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12716 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12717#endif
12718 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12719 AssertRCReturn(rc, rc);
12720
12721 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12722 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12723 VBOXSTRICTRC rcStrict2;
12724 switch (uAccessType)
12725 {
12726 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12727 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12728 {
12729 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12730 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12731 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12732
12733 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12734 GCPhys &= PAGE_BASE_GC_MASK;
12735 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12736 PVM pVM = pVCpu->CTX_SUFF(pVM);
12737 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12738 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12739
12740 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12741 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12742 CPUMCTX2CORE(pMixedCtx), GCPhys);
12743 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12744 if ( rcStrict2 == VINF_SUCCESS
12745 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12746 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12747 {
12748 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12749 | HM_CHANGED_GUEST_RSP
12750 | HM_CHANGED_GUEST_RFLAGS
12751 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12752 rcStrict2 = VINF_SUCCESS;
12753 }
12754 break;
12755 }
12756
12757 default:
12758 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12759 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12760 break;
12761 }
12762
12763 if (rcStrict2 != VINF_SUCCESS)
12764 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12765 return rcStrict2;
12766}
12767
12768
12769/**
12770 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12771 * VM-exit.
12772 */
12773HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12774{
12775 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12776
12777 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12778 if (pVmxTransient->fWasGuestDebugStateActive)
12779 {
12780 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12781 HMVMX_RETURN_UNEXPECTED_EXIT();
12782 }
12783
12784 if ( !pVCpu->hm.s.fSingleInstruction
12785 && !pVmxTransient->fWasHyperDebugStateActive)
12786 {
12787 Assert(!DBGFIsStepping(pVCpu));
12788 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12789
12790 /* Don't intercept MOV DRx any more. */
12791 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12792 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12793 AssertRCReturn(rc, rc);
12794
12795 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12796 VMMRZCallRing3Disable(pVCpu);
12797 HM_DISABLE_PREEMPT();
12798
12799 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12800 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12801 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12802
12803 HM_RESTORE_PREEMPT();
12804 VMMRZCallRing3Enable(pVCpu);
12805
12806#ifdef VBOX_WITH_STATISTICS
12807 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12808 AssertRCReturn(rc, rc);
12809 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12810 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12811 else
12812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12813#endif
12814 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12815 return VINF_SUCCESS;
12816 }
12817
12818 /*
12819 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12820 * Update the segment registers and DR7 from the CPU.
12821 */
12822 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12823 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12824 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12825 AssertRCReturn(rc, rc);
12826 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12827
12828 PVM pVM = pVCpu->CTX_SUFF(pVM);
12829 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12830 {
12831 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12832 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12833 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12834 if (RT_SUCCESS(rc))
12835 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12837 }
12838 else
12839 {
12840 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12841 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12842 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12844 }
12845
12846 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12847 if (RT_SUCCESS(rc))
12848 {
12849 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12850 AssertRCReturn(rc2, rc2);
12851 return VINF_SUCCESS;
12852 }
12853 return rc;
12854}
12855
12856
12857/**
12858 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12859 * Conditional VM-exit.
12860 */
12861HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12862{
12863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12864 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12865
12866 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12867 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12868 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12869 {
12870 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12871 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12872 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12873 {
12874 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12875 return VERR_EM_INTERPRETER;
12876 }
12877 }
12878 else
12879 {
12880 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12881 rcStrict1 = VINF_SUCCESS;
12882 return rcStrict1;
12883 }
12884
12885 RTGCPHYS GCPhys = 0;
12886 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12887
12888#if 0
12889 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12890#else
12891 /* Aggressive state sync. for now. */
12892 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12893 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12894 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12895#endif
12896 AssertRCReturn(rc, rc);
12897
12898 /*
12899 * If we succeed, resume guest execution.
12900 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12901 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12902 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12903 * weird case. See @bugref{6043}.
12904 */
12905 PVM pVM = pVCpu->CTX_SUFF(pVM);
12906 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12907 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12908 if ( rcStrict2 == VINF_SUCCESS
12909 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12910 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12911 {
12912 /* Successfully handled MMIO operation. */
12913 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12914 | HM_CHANGED_GUEST_RSP
12915 | HM_CHANGED_GUEST_RFLAGS
12916 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12917 return VINF_SUCCESS;
12918 }
12919 return rcStrict2;
12920}
12921
12922
12923/**
12924 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12925 * VM-exit.
12926 */
12927HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12928{
12929 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12930 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12931
12932 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12933 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12934 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12935 {
12936 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12937 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12938 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12939 }
12940 else
12941 {
12942 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12943 rcStrict1 = VINF_SUCCESS;
12944 return rcStrict1;
12945 }
12946
12947 RTGCPHYS GCPhys = 0;
12948 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12949 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12950#if 0
12951 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12952#else
12953 /* Aggressive state sync. for now. */
12954 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12955 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12956 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12957#endif
12958 AssertRCReturn(rc, rc);
12959
12960 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12961 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12962
12963 RTGCUINT uErrorCode = 0;
12964 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12965 uErrorCode |= X86_TRAP_PF_ID;
12966 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12967 uErrorCode |= X86_TRAP_PF_RW;
12968 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12969 uErrorCode |= X86_TRAP_PF_P;
12970
12971 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12972
12973 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12974 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12975
12976 /* Handle the pagefault trap for the nested shadow table. */
12977 PVM pVM = pVCpu->CTX_SUFF(pVM);
12978 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12979 TRPMResetTrap(pVCpu);
12980
12981 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12982 if ( rcStrict2 == VINF_SUCCESS
12983 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12984 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12985 {
12986 /* Successfully synced our nested page tables. */
12987 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12988 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12989 | HM_CHANGED_GUEST_RSP
12990 | HM_CHANGED_GUEST_RFLAGS);
12991 return VINF_SUCCESS;
12992 }
12993
12994 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12995 return rcStrict2;
12996}
12997
12998/** @} */
12999
13000/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13001/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13002/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13003
13004/** @name VM-exit exception handlers.
13005 * @{
13006 */
13007
13008/**
13009 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13010 */
13011static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13012{
13013 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13015
13016 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13017 AssertRCReturn(rc, rc);
13018
13019 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13020 {
13021 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13022 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13023
13024 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13025 * provides VM-exit instruction length. If this causes problem later,
13026 * disassemble the instruction like it's done on AMD-V. */
13027 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13028 AssertRCReturn(rc2, rc2);
13029 return rc;
13030 }
13031
13032 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13033 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13034 return rc;
13035}
13036
13037
13038/**
13039 * VM-exit exception handler for \#BP (Breakpoint exception).
13040 */
13041static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13042{
13043 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13045
13046 /** @todo Try optimize this by not saving the entire guest state unless
13047 * really needed. */
13048 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13049 AssertRCReturn(rc, rc);
13050
13051 PVM pVM = pVCpu->CTX_SUFF(pVM);
13052 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13053 if (rc == VINF_EM_RAW_GUEST_TRAP)
13054 {
13055 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13056 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13057 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13058 AssertRCReturn(rc, rc);
13059
13060 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13061 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13062 }
13063
13064 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13065 return rc;
13066}
13067
13068
13069/**
13070 * VM-exit exception handler for \#AC (alignment check exception).
13071 */
13072static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13073{
13074 RT_NOREF_PV(pMixedCtx);
13075 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13076
13077 /*
13078 * Re-inject it. We'll detect any nesting before getting here.
13079 */
13080 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13081 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13082 AssertRCReturn(rc, rc);
13083 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13084
13085 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13086 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13087 return VINF_SUCCESS;
13088}
13089
13090
13091/**
13092 * VM-exit exception handler for \#DB (Debug exception).
13093 */
13094static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13095{
13096 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13097 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13098 Log6(("XcptDB\n"));
13099
13100 /*
13101 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13102 * for processing.
13103 */
13104 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13105 AssertRCReturn(rc, rc);
13106
13107 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13108 uint64_t uDR6 = X86_DR6_INIT_VAL;
13109 uDR6 |= ( pVmxTransient->uExitQualification
13110 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13111
13112 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13113 if (rc == VINF_EM_RAW_GUEST_TRAP)
13114 {
13115 /*
13116 * The exception was for the guest. Update DR6, DR7.GD and
13117 * IA32_DEBUGCTL.LBR before forwarding it.
13118 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13119 */
13120 VMMRZCallRing3Disable(pVCpu);
13121 HM_DISABLE_PREEMPT();
13122
13123 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13124 pMixedCtx->dr[6] |= uDR6;
13125 if (CPUMIsGuestDebugStateActive(pVCpu))
13126 ASMSetDR6(pMixedCtx->dr[6]);
13127
13128 HM_RESTORE_PREEMPT();
13129 VMMRZCallRing3Enable(pVCpu);
13130
13131 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13132 AssertRCReturn(rc, rc);
13133
13134 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13135 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13136
13137 /* Paranoia. */
13138 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13139 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13140
13141 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13142 AssertRCReturn(rc, rc);
13143
13144 /*
13145 * Raise #DB in the guest.
13146 *
13147 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13148 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13149 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13150 *
13151 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13152 */
13153 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13154 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13155 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13156 AssertRCReturn(rc, rc);
13157 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13158 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13159 return VINF_SUCCESS;
13160 }
13161
13162 /*
13163 * Not a guest trap, must be a hypervisor related debug event then.
13164 * Update DR6 in case someone is interested in it.
13165 */
13166 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13167 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13168 CPUMSetHyperDR6(pVCpu, uDR6);
13169
13170 return rc;
13171}
13172
13173
13174/**
13175 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13176 * point exception).
13177 */
13178static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13179{
13180 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13181
13182 /* We require CR0 and EFER. EFER is always up-to-date. */
13183 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13184 AssertRCReturn(rc, rc);
13185
13186 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13187 VMMRZCallRing3Disable(pVCpu);
13188 HM_DISABLE_PREEMPT();
13189
13190 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13191 if (pVmxTransient->fWasGuestFPUStateActive)
13192 {
13193 rc = VINF_EM_RAW_GUEST_TRAP;
13194 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13195 }
13196 else
13197 {
13198#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13199 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13200#endif
13201 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13202 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13203 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13204 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13205 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13206 }
13207
13208 HM_RESTORE_PREEMPT();
13209 VMMRZCallRing3Enable(pVCpu);
13210
13211 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13212 {
13213 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13214 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13215 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13216 pVCpu->hm.s.fPreloadGuestFpu = true;
13217 }
13218 else
13219 {
13220 /* Forward #NM to the guest. */
13221 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13222 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13223 AssertRCReturn(rc, rc);
13224 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13225 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13226 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13227 }
13228
13229 return VINF_SUCCESS;
13230}
13231
13232
13233/**
13234 * VM-exit exception handler for \#GP (General-protection exception).
13235 *
13236 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13237 */
13238static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13239{
13240 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13241 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13242
13243 int rc;
13244 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13245 { /* likely */ }
13246 else
13247 {
13248#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13249 Assert(pVCpu->hm.s.fUsingDebugLoop);
13250#endif
13251 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13252 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13253 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13254 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13255 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13256 AssertRCReturn(rc, rc);
13257 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13258 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13259 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13260 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13261 return rc;
13262 }
13263
13264 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13265 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13266
13267 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13268 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13269 AssertRCReturn(rc, rc);
13270
13271 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13272 uint32_t cbOp = 0;
13273 PVM pVM = pVCpu->CTX_SUFF(pVM);
13274 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13275 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13276 if (RT_SUCCESS(rc))
13277 {
13278 rc = VINF_SUCCESS;
13279 Assert(cbOp == pDis->cbInstr);
13280 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13281 switch (pDis->pCurInstr->uOpcode)
13282 {
13283 case OP_CLI:
13284 {
13285 pMixedCtx->eflags.Bits.u1IF = 0;
13286 pMixedCtx->eflags.Bits.u1RF = 0;
13287 pMixedCtx->rip += pDis->cbInstr;
13288 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13289 if ( !fDbgStepping
13290 && pMixedCtx->eflags.Bits.u1TF)
13291 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13292 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13293 break;
13294 }
13295
13296 case OP_STI:
13297 {
13298 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13299 pMixedCtx->eflags.Bits.u1IF = 1;
13300 pMixedCtx->eflags.Bits.u1RF = 0;
13301 pMixedCtx->rip += pDis->cbInstr;
13302 if (!fOldIF)
13303 {
13304 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13305 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13306 }
13307 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13308 if ( !fDbgStepping
13309 && pMixedCtx->eflags.Bits.u1TF)
13310 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13311 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13312 break;
13313 }
13314
13315 case OP_HLT:
13316 {
13317 rc = VINF_EM_HALT;
13318 pMixedCtx->rip += pDis->cbInstr;
13319 pMixedCtx->eflags.Bits.u1RF = 0;
13320 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13322 break;
13323 }
13324
13325 case OP_POPF:
13326 {
13327 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13328 uint32_t cbParm;
13329 uint32_t uMask;
13330 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13331 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13332 {
13333 cbParm = 4;
13334 uMask = 0xffffffff;
13335 }
13336 else
13337 {
13338 cbParm = 2;
13339 uMask = 0xffff;
13340 }
13341
13342 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13343 RTGCPTR GCPtrStack = 0;
13344 X86EFLAGS Eflags;
13345 Eflags.u32 = 0;
13346 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13347 &GCPtrStack);
13348 if (RT_SUCCESS(rc))
13349 {
13350 Assert(sizeof(Eflags.u32) >= cbParm);
13351 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13352 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13353 }
13354 if (RT_FAILURE(rc))
13355 {
13356 rc = VERR_EM_INTERPRETER;
13357 break;
13358 }
13359 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13360 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13361 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13362 pMixedCtx->esp += cbParm;
13363 pMixedCtx->esp &= uMask;
13364 pMixedCtx->rip += pDis->cbInstr;
13365 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13366 | HM_CHANGED_GUEST_RSP
13367 | HM_CHANGED_GUEST_RFLAGS);
13368 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13369 POPF restores EFLAGS.TF. */
13370 if ( !fDbgStepping
13371 && fGstStepping)
13372 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13373 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13374 break;
13375 }
13376
13377 case OP_PUSHF:
13378 {
13379 uint32_t cbParm;
13380 uint32_t uMask;
13381 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13382 {
13383 cbParm = 4;
13384 uMask = 0xffffffff;
13385 }
13386 else
13387 {
13388 cbParm = 2;
13389 uMask = 0xffff;
13390 }
13391
13392 /* Get the stack pointer & push the contents of eflags onto the stack. */
13393 RTGCPTR GCPtrStack = 0;
13394 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13395 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13396 if (RT_FAILURE(rc))
13397 {
13398 rc = VERR_EM_INTERPRETER;
13399 break;
13400 }
13401 X86EFLAGS Eflags = pMixedCtx->eflags;
13402 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13403 Eflags.Bits.u1RF = 0;
13404 Eflags.Bits.u1VM = 0;
13405
13406 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13407 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13408 {
13409 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13410 rc = VERR_EM_INTERPRETER;
13411 break;
13412 }
13413 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13414 pMixedCtx->esp -= cbParm;
13415 pMixedCtx->esp &= uMask;
13416 pMixedCtx->rip += pDis->cbInstr;
13417 pMixedCtx->eflags.Bits.u1RF = 0;
13418 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13419 | HM_CHANGED_GUEST_RSP
13420 | HM_CHANGED_GUEST_RFLAGS);
13421 if ( !fDbgStepping
13422 && pMixedCtx->eflags.Bits.u1TF)
13423 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13424 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13425 break;
13426 }
13427
13428 case OP_IRET:
13429 {
13430 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13431 * instruction reference. */
13432 RTGCPTR GCPtrStack = 0;
13433 uint32_t uMask = 0xffff;
13434 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13435 uint16_t aIretFrame[3];
13436 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13437 {
13438 rc = VERR_EM_INTERPRETER;
13439 break;
13440 }
13441 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13442 &GCPtrStack);
13443 if (RT_SUCCESS(rc))
13444 {
13445 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13446 PGMACCESSORIGIN_HM));
13447 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13448 }
13449 if (RT_FAILURE(rc))
13450 {
13451 rc = VERR_EM_INTERPRETER;
13452 break;
13453 }
13454 pMixedCtx->eip = 0;
13455 pMixedCtx->ip = aIretFrame[0];
13456 pMixedCtx->cs.Sel = aIretFrame[1];
13457 pMixedCtx->cs.ValidSel = aIretFrame[1];
13458 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13459 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13460 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13461 pMixedCtx->sp += sizeof(aIretFrame);
13462 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13463 | HM_CHANGED_GUEST_SEGMENT_REGS
13464 | HM_CHANGED_GUEST_RSP
13465 | HM_CHANGED_GUEST_RFLAGS);
13466 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13467 if ( !fDbgStepping
13468 && fGstStepping)
13469 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13470 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13472 break;
13473 }
13474
13475 case OP_INT:
13476 {
13477 uint16_t uVector = pDis->Param1.uValue & 0xff;
13478 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13479 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13481 break;
13482 }
13483
13484 case OP_INTO:
13485 {
13486 if (pMixedCtx->eflags.Bits.u1OF)
13487 {
13488 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13489 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13490 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13491 }
13492 else
13493 {
13494 pMixedCtx->eflags.Bits.u1RF = 0;
13495 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13496 }
13497 break;
13498 }
13499
13500 default:
13501 {
13502 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13503 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13504 EMCODETYPE_SUPERVISOR);
13505 rc = VBOXSTRICTRC_VAL(rc2);
13506 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13507 /** @todo We have to set pending-debug exceptions here when the guest is
13508 * single-stepping depending on the instruction that was interpreted. */
13509 Log4(("#GP rc=%Rrc\n", rc));
13510 break;
13511 }
13512 }
13513 }
13514 else
13515 rc = VERR_EM_INTERPRETER;
13516
13517 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13518 ("#GP Unexpected rc=%Rrc\n", rc));
13519 return rc;
13520}
13521
13522
13523/**
13524 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13525 * the exception reported in the VMX transient structure back into the VM.
13526 *
13527 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13528 * up-to-date.
13529 */
13530static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13531{
13532 RT_NOREF_PV(pMixedCtx);
13533 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13534#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13535 Assert(pVCpu->hm.s.fUsingDebugLoop);
13536#endif
13537
13538 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13539 hmR0VmxCheckExitDueToEventDelivery(). */
13540 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13541 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13542 AssertRCReturn(rc, rc);
13543 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13544
13545#ifdef DEBUG_ramshankar
13546 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13547 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13548 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13549#endif
13550
13551 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13552 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13553 return VINF_SUCCESS;
13554}
13555
13556
13557/**
13558 * VM-exit exception handler for \#PF (Page-fault exception).
13559 */
13560static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13561{
13562 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13563 PVM pVM = pVCpu->CTX_SUFF(pVM);
13564 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13565 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13566 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13567 AssertRCReturn(rc, rc);
13568
13569 if (!pVM->hm.s.fNestedPaging)
13570 { /* likely */ }
13571 else
13572 {
13573#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13574 Assert(pVCpu->hm.s.fUsingDebugLoop);
13575#endif
13576 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13577 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13578 {
13579 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13580 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13581 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13582 }
13583 else
13584 {
13585 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13586 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13587 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13588 }
13589 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13590 return rc;
13591 }
13592
13593 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13594 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13595 if (pVmxTransient->fVectoringPF)
13596 {
13597 Assert(pVCpu->hm.s.Event.fPending);
13598 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13599 }
13600
13601 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13602 AssertRCReturn(rc, rc);
13603
13604 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13605 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13606
13607 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13608 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13609 (RTGCPTR)pVmxTransient->uExitQualification);
13610
13611 Log4(("#PF: rc=%Rrc\n", rc));
13612 if (rc == VINF_SUCCESS)
13613 {
13614#if 0
13615 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13616 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13617 * memory? We don't update the whole state here... */
13618 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13619 | HM_CHANGED_GUEST_RSP
13620 | HM_CHANGED_GUEST_RFLAGS
13621 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13622#else
13623 /*
13624 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13625 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13626 */
13627 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13628 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13629#endif
13630 TRPMResetTrap(pVCpu);
13631 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13632 return rc;
13633 }
13634
13635 if (rc == VINF_EM_RAW_GUEST_TRAP)
13636 {
13637 if (!pVmxTransient->fVectoringDoublePF)
13638 {
13639 /* It's a guest page fault and needs to be reflected to the guest. */
13640 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13641 TRPMResetTrap(pVCpu);
13642 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13643 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13644 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13645 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13646 }
13647 else
13648 {
13649 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13650 TRPMResetTrap(pVCpu);
13651 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13652 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13653 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13654 }
13655
13656 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13657 return VINF_SUCCESS;
13658 }
13659
13660 TRPMResetTrap(pVCpu);
13661 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13662 return rc;
13663}
13664
13665/** @} */
13666
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette