VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 64881

Last change on this file since 64881 was 64881, checked in by vboxsync, 8 years ago

VMM/HMVMXR0: Remove the Vista 64-bit TPR-threshold hack.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 584.9 KB
Line 
1/* $Id: HMVMXR0.cpp 64881 2016-12-15 14:56:03Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#ifdef DEBUG_ramshankar
44# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
45# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
46# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_CHECK_GUEST_STATE
48# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
49# define HMVMX_ALWAYS_TRAP_PF
50# define HMVMX_ALWAYS_SWAP_FPU_STATE
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DEBUG \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
142 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
143 * due to bugs in Intel CPUs.
144 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
145 * support.
146 */
147#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
148 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
149 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
150 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
151 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
152 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
153 | RT_BIT(X86_XCPT_XF))
154
155/**
156 * Exception bitmap mask for all contributory exceptions.
157 *
158 * Page fault is deliberately excluded here as it's conditional as to whether
159 * it's contributory or benign. Page faults are handled separately.
160 */
161#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
162 | RT_BIT(X86_XCPT_DE))
163
164/** Maximum VM-instruction error number. */
165#define HMVMX_INSTR_ERROR_MAX 28
166
167/** Profiling macro. */
168#ifdef HM_PROFILE_EXIT_DISPATCH
169# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
170# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
171#else
172# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
174#endif
175
176/** Assert that preemption is disabled or covered by thread-context hooks. */
177#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
178 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
179
180/** Assert that we haven't migrated CPUs when thread-context hooks are not
181 * used. */
182#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
183 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
184 ("Illegal migration! Entered on CPU %u Current %u\n", \
185 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
186
187/** Helper macro for VM-exit handlers called unexpectedly. */
188#define HMVMX_RETURN_UNEXPECTED_EXIT() \
189 do { \
190 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
191 return VERR_VMX_UNEXPECTED_EXIT; \
192 } while (0)
193
194
195/*********************************************************************************************************************************
196* Structures and Typedefs *
197*********************************************************************************************************************************/
198/**
199 * VMX transient state.
200 *
201 * A state structure for holding miscellaneous information across
202 * VMX non-root operation and restored after the transition.
203 */
204typedef struct VMXTRANSIENT
205{
206 /** The host's rflags/eflags. */
207 RTCCUINTREG fEFlags;
208#if HC_ARCH_BITS == 32
209 uint32_t u32Alignment0;
210#endif
211 /** The guest's TPR value used for TPR shadowing. */
212 uint8_t u8GuestTpr;
213 /** Alignment. */
214 uint8_t abAlignment0[7];
215
216 /** The basic VM-exit reason. */
217 uint16_t uExitReason;
218 /** Alignment. */
219 uint16_t u16Alignment0;
220 /** The VM-exit interruption error code. */
221 uint32_t uExitIntErrorCode;
222 /** The VM-exit exit code qualification. */
223 uint64_t uExitQualification;
224
225 /** The VM-exit interruption-information field. */
226 uint32_t uExitIntInfo;
227 /** The VM-exit instruction-length field. */
228 uint32_t cbInstr;
229 /** The VM-exit instruction-information field. */
230 union
231 {
232 /** Plain unsigned int representation. */
233 uint32_t u;
234 /** INS and OUTS information. */
235 struct
236 {
237 uint32_t u7Reserved0 : 7;
238 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
239 uint32_t u3AddrSize : 3;
240 uint32_t u5Reserved1 : 5;
241 /** The segment register (X86_SREG_XXX). */
242 uint32_t iSegReg : 3;
243 uint32_t uReserved2 : 14;
244 } StrIo;
245 } ExitInstrInfo;
246 /** Whether the VM-entry failed or not. */
247 bool fVMEntryFailed;
248 /** Alignment. */
249 uint8_t abAlignment1[3];
250
251 /** The VM-entry interruption-information field. */
252 uint32_t uEntryIntInfo;
253 /** The VM-entry exception error code field. */
254 uint32_t uEntryXcptErrorCode;
255 /** The VM-entry instruction length field. */
256 uint32_t cbEntryInstr;
257
258 /** IDT-vectoring information field. */
259 uint32_t uIdtVectoringInfo;
260 /** IDT-vectoring error code. */
261 uint32_t uIdtVectoringErrorCode;
262
263 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
264 uint32_t fVmcsFieldsRead;
265
266 /** Whether the guest FPU was active at the time of VM-exit. */
267 bool fWasGuestFPUStateActive;
268 /** Whether the guest debug state was active at the time of VM-exit. */
269 bool fWasGuestDebugStateActive;
270 /** Whether the hyper debug state was active at the time of VM-exit. */
271 bool fWasHyperDebugStateActive;
272 /** Whether TSC-offsetting should be setup before VM-entry. */
273 bool fUpdateTscOffsettingAndPreemptTimer;
274 /** Whether the VM-exit was caused by a page-fault during delivery of a
275 * contributory exception or a page-fault. */
276 bool fVectoringDoublePF;
277 /** Whether the VM-exit was caused by a page-fault during delivery of an
278 * external interrupt or NMI. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns Strict VBox status code (i.e. informational status codes too).
321 * @param pVCpu The cross context virtual CPU structure.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335/**
336 * VMX VM-exit handler, non-strict status code.
337 *
338 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
339 *
340 * @returns VBox status code, no informational status code returned.
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 *
347 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
348 * use of that status code will be replaced with VINF_EM_SOMETHING
349 * later when switching over to IEM.
350 */
351#ifndef HMVMX_USE_FUNCTION_TABLE
352typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
353#else
354typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
355#endif
356
357
358/*********************************************************************************************************************************
359* Internal Functions *
360*********************************************************************************************************************************/
361static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
362static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
363static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
364static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
365 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
366 bool fStepping, uint32_t *puIntState);
367#if HC_ARCH_BITS == 32
368static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
369#endif
370#ifndef HMVMX_USE_FUNCTION_TABLE
371DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
372# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
373# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
374#else
375# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
376# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
377#endif
378
379
380/** @name VM-exit handlers.
381 * @{
382 */
383static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
384static FNVMXEXITHANDLER hmR0VmxExitExtInt;
385static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
386static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
387static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
392static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
393static FNVMXEXITHANDLER hmR0VmxExitCpuid;
394static FNVMXEXITHANDLER hmR0VmxExitGetsec;
395static FNVMXEXITHANDLER hmR0VmxExitHlt;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
397static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
398static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
399static FNVMXEXITHANDLER hmR0VmxExitVmcall;
400static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
403static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
404static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
405static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
406static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
407static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
411static FNVMXEXITHANDLER hmR0VmxExitMwait;
412static FNVMXEXITHANDLER hmR0VmxExitMtf;
413static FNVMXEXITHANDLER hmR0VmxExitMonitor;
414static FNVMXEXITHANDLER hmR0VmxExitPause;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
417static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
418static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
419static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
420static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
421static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
422static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
423static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
425static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
426static FNVMXEXITHANDLER hmR0VmxExitRdrand;
427static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
428/** @} */
429
430static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
431static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
432static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
439
440
441/*********************************************************************************************************************************
442* Global Variables *
443*********************************************************************************************************************************/
444#ifdef HMVMX_USE_FUNCTION_TABLE
445
446/**
447 * VMX_EXIT dispatch table.
448 */
449static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
450{
451 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
452 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
453 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
454 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
455 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
456 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
457 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
458 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
459 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
460 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
461 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
462 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
463 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
464 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
465 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
466 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
467 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
468 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
469 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
470 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
471 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
472 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
473 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
474 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
475 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
476 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
477 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
478 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
479 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
480 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
481 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
482 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
483 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
484 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
485 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
486 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
487 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
488 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
489 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
490 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
491 /* 40 UNDEFINED */ hmR0VmxExitPause,
492 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
493 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
494 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
495 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
496 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
498 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
499 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
500 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
501 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
502 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
503 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
504 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
505 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
506 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
507 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
508 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
509 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
510 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
511 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
512 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
513 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
514 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
515 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
516};
517#endif /* HMVMX_USE_FUNCTION_TABLE */
518
519#ifdef VBOX_STRICT
520static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
521{
522 /* 0 */ "(Not Used)",
523 /* 1 */ "VMCALL executed in VMX root operation.",
524 /* 2 */ "VMCLEAR with invalid physical address.",
525 /* 3 */ "VMCLEAR with VMXON pointer.",
526 /* 4 */ "VMLAUNCH with non-clear VMCS.",
527 /* 5 */ "VMRESUME with non-launched VMCS.",
528 /* 6 */ "VMRESUME after VMXOFF",
529 /* 7 */ "VM-entry with invalid control fields.",
530 /* 8 */ "VM-entry with invalid host state fields.",
531 /* 9 */ "VMPTRLD with invalid physical address.",
532 /* 10 */ "VMPTRLD with VMXON pointer.",
533 /* 11 */ "VMPTRLD with incorrect revision identifier.",
534 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
535 /* 13 */ "VMWRITE to read-only VMCS component.",
536 /* 14 */ "(Not Used)",
537 /* 15 */ "VMXON executed in VMX root operation.",
538 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
539 /* 17 */ "VM-entry with non-launched executing VMCS.",
540 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
541 /* 19 */ "VMCALL with non-clear VMCS.",
542 /* 20 */ "VMCALL with invalid VM-exit control fields.",
543 /* 21 */ "(Not Used)",
544 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
545 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
546 /* 24 */ "VMCALL with invalid SMM-monitor features.",
547 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
548 /* 26 */ "VM-entry with events blocked by MOV SS.",
549 /* 27 */ "(Not Used)",
550 /* 28 */ "Invalid operand to INVEPT/INVVPID."
551};
552#endif /* VBOX_STRICT */
553
554
555
556/**
557 * Updates the VM's last error record.
558 *
559 * If there was a VMX instruction error, reads the error data from the VMCS and
560 * updates VCPU's last error record as well.
561 *
562 * @param pVM The cross context VM structure.
563 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
564 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
565 * VERR_VMX_INVALID_VMCS_FIELD.
566 * @param rc The error code.
567 */
568static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
569{
570 AssertPtr(pVM);
571 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
572 || rc == VERR_VMX_UNABLE_TO_START_VM)
573 {
574 AssertPtrReturnVoid(pVCpu);
575 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
576 }
577 pVM->hm.s.lLastError = rc;
578}
579
580
581/**
582 * Reads the VM-entry interruption-information field from the VMCS into the VMX
583 * transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598#ifdef VBOX_STRICT
599/**
600 * Reads the VM-entry exception error code field from the VMCS into
601 * the VMX transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 *
606 * @remarks No-long-jump zone!!!
607 */
608DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609{
610 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
611 AssertRCReturn(rc, rc);
612 return VINF_SUCCESS;
613}
614#endif /* VBOX_STRICT */
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc, rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
923 }
924
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
926#ifdef VBOX_WITH_CRASHDUMP_MAGIC
927 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
928#endif
929}
930
931
932/**
933 * Worker function to allocate VT-x related VM structures.
934 *
935 * @returns IPRT status code.
936 * @param pVM The cross context VM structure.
937 */
938static int hmR0VmxStructsAlloc(PVM pVM)
939{
940 /*
941 * Initialize members up-front so we can cleanup properly on allocation failure.
942 */
943#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
944 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
945 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
946 pVM->hm.s.vmx.HCPhys##a_Name = 0;
947
948#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
952
953#ifdef VBOX_WITH_CRASHDUMP_MAGIC
954 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
955#endif
956 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
957
958 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
959 for (VMCPUID i = 0; i < pVM->cCpus; i++)
960 {
961 PVMCPU pVCpu = &pVM->aCpus[i];
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
965 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
967 }
968#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
969#undef VMXLOCAL_INIT_VM_MEMOBJ
970
971 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
972 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
973 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
974 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
975
976 /*
977 * Allocate all the VT-x structures.
978 */
979 int rc = VINF_SUCCESS;
980#ifdef VBOX_WITH_CRASHDUMP_MAGIC
981 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
982 if (RT_FAILURE(rc))
983 goto cleanup;
984 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
985 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
986#endif
987
988 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
989 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
990 {
991 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
992 &pVM->hm.s.vmx.HCPhysApicAccess);
993 if (RT_FAILURE(rc))
994 goto cleanup;
995 }
996
997 /*
998 * Initialize per-VCPU VT-x structures.
999 */
1000 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1001 {
1002 PVMCPU pVCpu = &pVM->aCpus[i];
1003 AssertPtr(pVCpu);
1004
1005 /* Allocate the VM control structure (VMCS). */
1006 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1007 if (RT_FAILURE(rc))
1008 goto cleanup;
1009
1010 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1011 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1012 {
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1014 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1015 if (RT_FAILURE(rc))
1016 goto cleanup;
1017 }
1018
1019 /*
1020 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1021 * transparent accesses of specific MSRs.
1022 *
1023 * If the condition for enabling MSR bitmaps changes here, don't forget to
1024 * update HMAreMsrBitmapsAvailable().
1025 */
1026 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1027 {
1028 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1029 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1030 if (RT_FAILURE(rc))
1031 goto cleanup;
1032 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1033 }
1034
1035 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1037 if (RT_FAILURE(rc))
1038 goto cleanup;
1039
1040 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1041 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044 }
1045
1046 return VINF_SUCCESS;
1047
1048cleanup:
1049 hmR0VmxStructsFree(pVM);
1050 return rc;
1051}
1052
1053
1054/**
1055 * Does global VT-x initialization (called during module initialization).
1056 *
1057 * @returns VBox status code.
1058 */
1059VMMR0DECL(int) VMXR0GlobalInit(void)
1060{
1061#ifdef HMVMX_USE_FUNCTION_TABLE
1062 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1063# ifdef VBOX_STRICT
1064 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1065 Assert(g_apfnVMExitHandlers[i]);
1066# endif
1067#endif
1068 return VINF_SUCCESS;
1069}
1070
1071
1072/**
1073 * Does global VT-x termination (called during module termination).
1074 */
1075VMMR0DECL(void) VMXR0GlobalTerm()
1076{
1077 /* Nothing to do currently. */
1078}
1079
1080
1081/**
1082 * Sets up and activates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pVM The cross context VM structure. Can be
1087 * NULL after a host resume operation.
1088 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1089 * fEnabledByHost is @c true).
1090 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1091 * @a fEnabledByHost is @c true).
1092 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1093 * enable VT-x on the host.
1094 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1095 */
1096VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1097 void *pvMsrs)
1098{
1099 Assert(pCpu);
1100 Assert(pvMsrs);
1101 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102
1103 /* Enable VT-x if it's not already enabled by the host. */
1104 if (!fEnabledByHost)
1105 {
1106 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1107 if (RT_FAILURE(rc))
1108 return rc;
1109 }
1110
1111 /*
1112 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1113 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1119 pCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 NOREF(pCpu);
1145 NOREF(pvCpuPage);
1146 NOREF(HCPhysCpuPage);
1147
1148 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1149 return hmR0VmxLeaveRootMode();
1150}
1151
1152
1153/**
1154 * Sets the permission bits for the specified MSR in the MSR bitmap.
1155 *
1156 * @param pVCpu The cross context virtual CPU structure.
1157 * @param uMsr The MSR value.
1158 * @param enmRead Whether reading this MSR causes a VM-exit.
1159 * @param enmWrite Whether writing this MSR causes a VM-exit.
1160 */
1161static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1162{
1163 int32_t iBit;
1164 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1165
1166 /*
1167 * Layout:
1168 * 0x000 - 0x3ff - Low MSR read bits
1169 * 0x400 - 0x7ff - High MSR read bits
1170 * 0x800 - 0xbff - Low MSR write bits
1171 * 0xc00 - 0xfff - High MSR write bits
1172 */
1173 if (uMsr <= 0x00001FFF)
1174 iBit = uMsr;
1175 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1176 {
1177 iBit = uMsr - UINT32_C(0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1182
1183 Assert(iBit <= 0x1fff);
1184 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1185 ASMBitSet(pbMsrBitmap, iBit);
1186 else
1187 ASMBitClear(pbMsrBitmap, iBit);
1188
1189 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1190 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1191 else
1192 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1193}
1194
1195
1196#ifdef VBOX_STRICT
1197/**
1198 * Gets the permission bits for the specified MSR in the MSR bitmap.
1199 *
1200 * @returns VBox status code.
1201 * @retval VINF_SUCCESS if the specified MSR is found.
1202 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1203 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1204 *
1205 * @param pVCpu The cross context virtual CPU structure.
1206 * @param uMsr The MSR.
1207 * @param penmRead Where to store the read permissions.
1208 * @param penmWrite Where to store the write permissions.
1209 */
1210static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1211{
1212 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1213 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1214 int32_t iBit;
1215 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1216
1217 /* See hmR0VmxSetMsrPermission() for the layout. */
1218 if (uMsr <= 0x00001FFF)
1219 iBit = uMsr;
1220 else if ( uMsr >= 0xC0000000
1221 && uMsr <= 0xC0001FFF)
1222 {
1223 iBit = (uMsr - 0xC0000000);
1224 pbMsrBitmap += 0x400;
1225 }
1226 else
1227 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1228
1229 Assert(iBit <= 0x1fff);
1230 if (ASMBitTest(pbMsrBitmap, iBit))
1231 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1232 else
1233 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1234
1235 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1236 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1237 else
1238 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1239 return VINF_SUCCESS;
1240}
1241#endif /* VBOX_STRICT */
1242
1243
1244/**
1245 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1246 * area.
1247 *
1248 * @returns VBox status code.
1249 * @param pVCpu The cross context virtual CPU structure.
1250 * @param cMsrs The number of MSRs.
1251 */
1252DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1253{
1254 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1255 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1256 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1257 {
1258 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1259 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1261 }
1262
1263 /* Update number of guest MSRs to load/store across the world-switch. */
1264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1265 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1266
1267 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1269 AssertRCReturn(rc, rc);
1270
1271 /* Update the VCPU's copy of the MSR count. */
1272 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1273
1274 return VINF_SUCCESS;
1275}
1276
1277
1278/**
1279 * Adds a new (or updates the value of an existing) guest/host MSR
1280 * pair to be swapped during the world-switch as part of the
1281 * auto-load/store MSR area in the VMCS.
1282 *
1283 * @returns VBox status code.
1284 * @param pVCpu The cross context virtual CPU structure.
1285 * @param uMsr The MSR.
1286 * @param uGuestMsrValue Value of the guest MSR.
1287 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1288 * necessary.
1289 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1290 * its value was updated. Optional, can be NULL.
1291 */
1292static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1293 bool *pfAddedAndUpdated)
1294{
1295 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1296 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1297 uint32_t i;
1298 for (i = 0; i < cMsrs; i++)
1299 {
1300 if (pGuestMsr->u32Msr == uMsr)
1301 break;
1302 pGuestMsr++;
1303 }
1304
1305 bool fAdded = false;
1306 if (i == cMsrs)
1307 {
1308 ++cMsrs;
1309 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1310 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1311
1312 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1313 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1314 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1315
1316 fAdded = true;
1317 }
1318
1319 /* Update the MSR values in the auto-load/store MSR area. */
1320 pGuestMsr->u32Msr = uMsr;
1321 pGuestMsr->u64Value = uGuestMsrValue;
1322
1323 /* Create/update the MSR slot in the host MSR area. */
1324 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1325 pHostMsr += i;
1326 pHostMsr->u32Msr = uMsr;
1327
1328 /*
1329 * Update the host MSR only when requested by the caller AND when we're
1330 * adding it to the auto-load/store area. Otherwise, it would have been
1331 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1332 */
1333 bool fUpdatedMsrValue = false;
1334 if ( fAdded
1335 && fUpdateHostMsr)
1336 {
1337 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1338 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1339 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1340 fUpdatedMsrValue = true;
1341 }
1342
1343 if (pfAddedAndUpdated)
1344 *pfAddedAndUpdated = fUpdatedMsrValue;
1345 return VINF_SUCCESS;
1346}
1347
1348
1349/**
1350 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1351 * auto-load/store MSR area in the VMCS.
1352 *
1353 * @returns VBox status code.
1354 * @param pVCpu The cross context virtual CPU structure.
1355 * @param uMsr The MSR.
1356 */
1357static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1358{
1359 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1360 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1361 for (uint32_t i = 0; i < cMsrs; i++)
1362 {
1363 /* Find the MSR. */
1364 if (pGuestMsr->u32Msr == uMsr)
1365 {
1366 /* If it's the last MSR, simply reduce the count. */
1367 if (i == cMsrs - 1)
1368 {
1369 --cMsrs;
1370 break;
1371 }
1372
1373 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1374 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1375 pLastGuestMsr += cMsrs - 1;
1376 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1377 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1378
1379 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1381 pLastHostMsr += cMsrs - 1;
1382 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1383 pHostMsr->u64Value = pLastHostMsr->u64Value;
1384 --cMsrs;
1385 break;
1386 }
1387 pGuestMsr++;
1388 }
1389
1390 /* Update the VMCS if the count changed (meaning the MSR was found). */
1391 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1392 {
1393 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1394 AssertRCReturn(rc, rc);
1395
1396 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1397 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1398 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1399
1400 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1401 return VINF_SUCCESS;
1402 }
1403
1404 return VERR_NOT_FOUND;
1405}
1406
1407
1408/**
1409 * Checks if the specified guest MSR is part of the auto-load/store area in
1410 * the VMCS.
1411 *
1412 * @returns true if found, false otherwise.
1413 * @param pVCpu The cross context virtual CPU structure.
1414 * @param uMsr The MSR to find.
1415 */
1416static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1417{
1418 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1419 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1420
1421 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1422 {
1423 if (pGuestMsr->u32Msr == uMsr)
1424 return true;
1425 }
1426 return false;
1427}
1428
1429
1430/**
1431 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1432 *
1433 * @param pVCpu The cross context virtual CPU structure.
1434 *
1435 * @remarks No-long-jump zone!!!
1436 */
1437static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1438{
1439 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1440 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1441 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1442 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1443
1444 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1445 {
1446 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1447
1448 /*
1449 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1450 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1451 */
1452 if (pHostMsr->u32Msr == MSR_K6_EFER)
1453 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1454 else
1455 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1456 }
1457
1458 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1459}
1460
1461
1462/**
1463 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1464 * perform lazy restoration of the host MSRs while leaving VT-x.
1465 *
1466 * @param pVCpu The cross context virtual CPU structure.
1467 *
1468 * @remarks No-long-jump zone!!!
1469 */
1470static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1471{
1472 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1473
1474 /*
1475 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1476 */
1477 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1478 {
1479 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1480#if HC_ARCH_BITS == 64
1481 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1482 {
1483 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1484 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1485 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1486 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1487 }
1488#endif
1489 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1490 }
1491}
1492
1493
1494/**
1495 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1496 * lazily while leaving VT-x.
1497 *
1498 * @returns true if it does, false otherwise.
1499 * @param pVCpu The cross context virtual CPU structure.
1500 * @param uMsr The MSR to check.
1501 */
1502static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1503{
1504 NOREF(pVCpu);
1505#if HC_ARCH_BITS == 64
1506 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1507 {
1508 switch (uMsr)
1509 {
1510 case MSR_K8_LSTAR:
1511 case MSR_K6_STAR:
1512 case MSR_K8_SF_MASK:
1513 case MSR_K8_KERNEL_GS_BASE:
1514 return true;
1515 }
1516 }
1517#else
1518 RT_NOREF(pVCpu, uMsr);
1519#endif
1520 return false;
1521}
1522
1523
1524/**
1525 * Saves a set of guest MSRs back into the guest-CPU context.
1526 *
1527 * @param pVCpu The cross context virtual CPU structure.
1528 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1529 * out-of-sync. Make sure to update the required fields
1530 * before using them.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1540 {
1541 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1542#if HC_ARCH_BITS == 64
1543 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1544 {
1545 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1546 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1547 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1548 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1549 }
1550#else
1551 NOREF(pMixedCtx);
1552#endif
1553 }
1554}
1555
1556
1557/**
1558 * Loads a set of guests MSRs to allow read/passthru to the guest.
1559 *
1560 * The name of this function is slightly confusing. This function does NOT
1561 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1562 * common prefix for functions dealing with "lazy restoration" of the shared
1563 * MSRs.
1564 *
1565 * @param pVCpu The cross context virtual CPU structure.
1566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1567 * out-of-sync. Make sure to update the required fields
1568 * before using them.
1569 *
1570 * @remarks No-long-jump zone!!!
1571 */
1572static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1573{
1574 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1575 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1576
1577#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1578 do { \
1579 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1580 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1581 else \
1582 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1583 } while (0)
1584
1585 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1586 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1587 {
1588#if HC_ARCH_BITS == 64
1589 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1590 {
1591 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1592 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1593 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1594 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1595 }
1596#else
1597 RT_NOREF(pMixedCtx);
1598#endif
1599 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1600 }
1601
1602#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1603}
1604
1605
1606/**
1607 * Performs lazy restoration of the set of host MSRs if they were previously
1608 * loaded with guest MSR values.
1609 *
1610 * @param pVCpu The cross context virtual CPU structure.
1611 *
1612 * @remarks No-long-jump zone!!!
1613 * @remarks The guest MSRs should have been saved back into the guest-CPU
1614 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1615 */
1616static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1617{
1618 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1619 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1620
1621 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1622 {
1623 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1624#if HC_ARCH_BITS == 64
1625 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1626 {
1627 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1628 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1629 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1630 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1631 }
1632#endif
1633 }
1634 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1635}
1636
1637
1638/**
1639 * Verifies that our cached values of the VMCS controls are all
1640 * consistent with what's actually present in the VMCS.
1641 *
1642 * @returns VBox status code.
1643 * @param pVCpu The cross context virtual CPU structure.
1644 */
1645static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1646{
1647 uint32_t u32Val;
1648 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1651 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1652
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1656 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1657
1658 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1659 AssertRCReturn(rc, rc);
1660 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1661 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1662
1663 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1664 AssertRCReturn(rc, rc);
1665 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1666 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1667
1668 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1669 {
1670 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1671 AssertRCReturn(rc, rc);
1672 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1673 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1674 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1675 }
1676
1677 return VINF_SUCCESS;
1678}
1679
1680
1681#ifdef VBOX_STRICT
1682/**
1683 * Verifies that our cached host EFER value has not changed
1684 * since we cached it.
1685 *
1686 * @param pVCpu The cross context virtual CPU structure.
1687 */
1688static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1689{
1690 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1691
1692 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1693 {
1694 uint64_t u64Val;
1695 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1696 AssertRC(rc);
1697
1698 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1699 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1700 }
1701}
1702
1703
1704/**
1705 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1706 * VMCS are correct.
1707 *
1708 * @param pVCpu The cross context virtual CPU structure.
1709 */
1710static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1711{
1712 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1713
1714 /* Verify MSR counts in the VMCS are what we think it should be. */
1715 uint32_t cMsrs;
1716 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1717 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1718
1719 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1720 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1723 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1726 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1727 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1728 {
1729 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1730 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1731 pGuestMsr->u32Msr, cMsrs));
1732
1733 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1734 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1735 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1736
1737 /* Verify that the permissions are as expected in the MSR bitmap. */
1738 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1739 {
1740 VMXMSREXITREAD enmRead;
1741 VMXMSREXITWRITE enmWrite;
1742 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1743 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1744 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1745 {
1746 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1747 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1748 }
1749 else
1750 {
1751 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1752 pGuestMsr->u32Msr, cMsrs));
1753 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 }
1756 }
1757 }
1758}
1759#endif /* VBOX_STRICT */
1760
1761
1762/**
1763 * Flushes the TLB using EPT.
1764 *
1765 * @returns VBox status code.
1766 * @param pVCpu The cross context virtual CPU structure of the calling
1767 * EMT. Can be NULL depending on @a enmFlush.
1768 * @param enmFlush Type of flush.
1769 *
1770 * @remarks Caller is responsible for making sure this function is called only
1771 * when NestedPaging is supported and providing @a enmFlush that is
1772 * supported by the CPU.
1773 * @remarks Can be called with interrupts disabled.
1774 */
1775static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1776{
1777 uint64_t au64Descriptor[2];
1778 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1779 au64Descriptor[0] = 0;
1780 else
1781 {
1782 Assert(pVCpu);
1783 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1784 }
1785 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1786
1787 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1788 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1789 rc));
1790 if ( RT_SUCCESS(rc)
1791 && pVCpu)
1792 {
1793 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1794 }
1795}
1796
1797
1798/**
1799 * Flushes the TLB using VPID.
1800 *
1801 * @returns VBox status code.
1802 * @param pVM The cross context VM structure.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmFlush.
1805 * @param enmFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1812{
1813 NOREF(pVM);
1814 AssertPtr(pVM);
1815 Assert(pVM->hm.s.vmx.fVpid);
1816
1817 uint64_t au64Descriptor[2];
1818 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1819 {
1820 au64Descriptor[0] = 0;
1821 au64Descriptor[1] = 0;
1822 }
1823 else
1824 {
1825 AssertPtr(pVCpu);
1826 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1828 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1829 au64Descriptor[1] = GCPtr;
1830 }
1831
1832 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1833 AssertMsg(rc == VINF_SUCCESS,
1834 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1835 if ( RT_SUCCESS(rc)
1836 && pVCpu)
1837 {
1838 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1839 }
1840}
1841
1842
1843/**
1844 * Invalidates a guest page by guest virtual address. Only relevant for
1845 * EPT/VPID, otherwise there is nothing really to invalidate.
1846 *
1847 * @returns VBox status code.
1848 * @param pVM The cross context VM structure.
1849 * @param pVCpu The cross context virtual CPU structure.
1850 * @param GCVirt Guest virtual address of the page to invalidate.
1851 */
1852VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1853{
1854 AssertPtr(pVM);
1855 AssertPtr(pVCpu);
1856 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1857
1858 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1859 if (!fFlushPending)
1860 {
1861 /*
1862 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1863 * See @bugref{6043} and @bugref{6177}.
1864 *
1865 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1866 * function maybe called in a loop with individual addresses.
1867 */
1868 if (pVM->hm.s.vmx.fVpid)
1869 {
1870 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1871 {
1872 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1873 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1874 }
1875 else
1876 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1877 }
1878 else if (pVM->hm.s.fNestedPaging)
1879 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1880 }
1881
1882 return VINF_SUCCESS;
1883}
1884
1885
1886/**
1887 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1888 * otherwise there is nothing really to invalidate.
1889 *
1890 * @returns VBox status code.
1891 * @param pVM The cross context VM structure.
1892 * @param pVCpu The cross context virtual CPU structure.
1893 * @param GCPhys Guest physical address of the page to invalidate.
1894 */
1895VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1896{
1897 NOREF(pVM); NOREF(GCPhys);
1898 LogFlowFunc(("%RGp\n", GCPhys));
1899
1900 /*
1901 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1902 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1903 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1904 */
1905 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1906 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1907 return VINF_SUCCESS;
1908}
1909
1910
1911/**
1912 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1913 * case where neither EPT nor VPID is supported by the CPU.
1914 *
1915 * @param pVM The cross context VM structure.
1916 * @param pVCpu The cross context virtual CPU structure.
1917 * @param pCpu Pointer to the global HM struct.
1918 *
1919 * @remarks Called with interrupts disabled.
1920 */
1921static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1922{
1923 AssertPtr(pVCpu);
1924 AssertPtr(pCpu);
1925 NOREF(pVM);
1926
1927 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1928
1929 Assert(pCpu->idCpu != NIL_RTCPUID);
1930 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1931 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1932 pVCpu->hm.s.fForceTLBFlush = false;
1933 return;
1934}
1935
1936
1937/**
1938 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1939 *
1940 * @param pVM The cross context VM structure.
1941 * @param pVCpu The cross context virtual CPU structure.
1942 * @param pCpu Pointer to the global HM CPU struct.
1943 * @remarks All references to "ASID" in this function pertains to "VPID" in
1944 * Intel's nomenclature. The reason is, to avoid confusion in compare
1945 * statements since the host-CPU copies are named "ASID".
1946 *
1947 * @remarks Called with interrupts disabled.
1948 */
1949static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1950{
1951#ifdef VBOX_WITH_STATISTICS
1952 bool fTlbFlushed = false;
1953# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1954# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1955 if (!fTlbFlushed) \
1956 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1957 } while (0)
1958#else
1959# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1960# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1961#endif
1962
1963 AssertPtr(pVM);
1964 AssertPtr(pCpu);
1965 AssertPtr(pVCpu);
1966 Assert(pCpu->idCpu != NIL_RTCPUID);
1967
1968 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1969 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1970 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1971
1972 /*
1973 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1974 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1975 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1976 */
1977 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1978 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1979 {
1980 ++pCpu->uCurrentAsid;
1981 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1982 {
1983 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1984 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1985 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1986 }
1987
1988 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1989 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1990 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1991
1992 /*
1993 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1994 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1995 */
1996 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1997 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1998 HMVMX_SET_TAGGED_TLB_FLUSHED();
1999 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2000 }
2001
2002 /* Check for explicit TLB flushes. */
2003 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2004 {
2005 /*
2006 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2007 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2008 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2009 * but not guest-physical mappings.
2010 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2011 */
2012 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2013 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2014 HMVMX_SET_TAGGED_TLB_FLUSHED();
2015 }
2016
2017 pVCpu->hm.s.fForceTLBFlush = false;
2018 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2019
2020 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2021 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2022 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2023 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2024 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2025 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2026 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2027 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2028 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2029
2030 /* Update VMCS with the VPID. */
2031 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2032 AssertRC(rc);
2033
2034#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2035}
2036
2037
2038/**
2039 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2040 *
2041 * @returns VBox status code.
2042 * @param pVM The cross context VM structure.
2043 * @param pVCpu The cross context virtual CPU structure.
2044 * @param pCpu Pointer to the global HM CPU struct.
2045 *
2046 * @remarks Called with interrupts disabled.
2047 */
2048static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2049{
2050 AssertPtr(pVM);
2051 AssertPtr(pVCpu);
2052 AssertPtr(pCpu);
2053 Assert(pCpu->idCpu != NIL_RTCPUID);
2054 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2055 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2056
2057 /*
2058 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2059 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2060 */
2061 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2062 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2063 {
2064 pVCpu->hm.s.fForceTLBFlush = true;
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2066 }
2067
2068 /* Check for explicit TLB flushes. */
2069 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2070 {
2071 pVCpu->hm.s.fForceTLBFlush = true;
2072 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2073 }
2074
2075 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2076 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2077
2078 if (pVCpu->hm.s.fForceTLBFlush)
2079 {
2080 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2081 pVCpu->hm.s.fForceTLBFlush = false;
2082 }
2083}
2084
2085
2086/**
2087 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2088 *
2089 * @returns VBox status code.
2090 * @param pVM The cross context VM structure.
2091 * @param pVCpu The cross context virtual CPU structure.
2092 * @param pCpu Pointer to the global HM CPU struct.
2093 *
2094 * @remarks Called with interrupts disabled.
2095 */
2096static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2097{
2098 AssertPtr(pVM);
2099 AssertPtr(pVCpu);
2100 AssertPtr(pCpu);
2101 Assert(pCpu->idCpu != NIL_RTCPUID);
2102 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2103 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2104
2105 /*
2106 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2107 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2108 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2109 */
2110 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2111 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2112 {
2113 pVCpu->hm.s.fForceTLBFlush = true;
2114 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2115 }
2116
2117 /* Check for explicit TLB flushes. */
2118 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2119 {
2120 /*
2121 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2122 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2123 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2124 */
2125 pVCpu->hm.s.fForceTLBFlush = true;
2126 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2127 }
2128
2129 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2130 if (pVCpu->hm.s.fForceTLBFlush)
2131 {
2132 ++pCpu->uCurrentAsid;
2133 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2134 {
2135 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2136 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2137 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2138 }
2139
2140 pVCpu->hm.s.fForceTLBFlush = false;
2141 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2142 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2143 if (pCpu->fFlushAsidBeforeUse)
2144 {
2145 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2146 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2147 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2148 {
2149 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2150 pCpu->fFlushAsidBeforeUse = false;
2151 }
2152 else
2153 {
2154 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2155 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2156 }
2157 }
2158 }
2159
2160 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2161 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2162 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2163 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2164 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2165 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2166 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2167
2168 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2169 AssertRC(rc);
2170}
2171
2172
2173/**
2174 * Flushes the guest TLB entry based on CPU capabilities.
2175 *
2176 * @param pVCpu The cross context virtual CPU structure.
2177 * @param pCpu Pointer to the global HM CPU struct.
2178 */
2179DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2180{
2181#ifdef HMVMX_ALWAYS_FLUSH_TLB
2182 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2183#endif
2184 PVM pVM = pVCpu->CTX_SUFF(pVM);
2185 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2186 {
2187 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2188 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2189 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2190 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2191 default:
2192 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2193 break;
2194 }
2195
2196 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2197}
2198
2199
2200/**
2201 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2202 * TLB entries from the host TLB before VM-entry.
2203 *
2204 * @returns VBox status code.
2205 * @param pVM The cross context VM structure.
2206 */
2207static int hmR0VmxSetupTaggedTlb(PVM pVM)
2208{
2209 /*
2210 * Determine optimal flush type for Nested Paging.
2211 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2212 * guest execution (see hmR3InitFinalizeR0()).
2213 */
2214 if (pVM->hm.s.fNestedPaging)
2215 {
2216 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2217 {
2218 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2220 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2221 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2222 else
2223 {
2224 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* Make sure the write-back cacheable memory type for EPT is supported. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237
2238 /* EPT requires a page-walk length of 4. */
2239 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2240 {
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246 else
2247 {
2248 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2249 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2250 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2252 }
2253 }
2254
2255 /*
2256 * Determine optimal flush type for VPID.
2257 */
2258 if (pVM->hm.s.vmx.fVpid)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2261 {
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2264 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2266 else
2267 {
2268 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2272 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277 else
2278 {
2279 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2280 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282 pVM->hm.s.vmx.fVpid = false;
2283 }
2284 }
2285
2286 /*
2287 * Setup the handler for flushing tagged-TLBs.
2288 */
2289 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2291 else if (pVM->hm.s.fNestedPaging)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2293 else if (pVM->hm.s.vmx.fVpid)
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2295 else
2296 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2297 return VINF_SUCCESS;
2298}
2299
2300
2301/**
2302 * Sets up pin-based VM-execution controls in the VMCS.
2303 *
2304 * @returns VBox status code.
2305 * @param pVM The cross context VM structure.
2306 * @param pVCpu The cross context virtual CPU structure.
2307 */
2308static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2309{
2310 AssertPtr(pVM);
2311 AssertPtr(pVCpu);
2312
2313 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2314 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2315
2316 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2317 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2318
2319 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2320 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2321
2322 /* Enable the VMX preemption timer. */
2323 if (pVM->hm.s.vmx.fUsePreemptTimer)
2324 {
2325 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2326 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2327 }
2328
2329#if 0
2330 /* Enable posted-interrupt processing. */
2331 if (pVM->hm.s.fPostedIntrs)
2332 {
2333 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2334 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2335 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2336 }
2337#endif
2338
2339 if ((val & zap) != val)
2340 {
2341 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2342 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2343 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2344 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2345 }
2346
2347 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2348 AssertRCReturn(rc, rc);
2349
2350 pVCpu->hm.s.vmx.u32PinCtls = val;
2351 return rc;
2352}
2353
2354
2355/**
2356 * Sets up processor-based VM-execution controls in the VMCS.
2357 *
2358 * @returns VBox status code.
2359 * @param pVM The cross context VM structure.
2360 * @param pVCpu The cross context virtual CPU structure.
2361 */
2362static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2363{
2364 AssertPtr(pVM);
2365 AssertPtr(pVCpu);
2366
2367 int rc = VERR_INTERNAL_ERROR_5;
2368 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2369 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2370
2371 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2377 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2378
2379 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2380 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2381 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2382 {
2383 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2384 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2385 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386 }
2387
2388 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2389 if (!pVM->hm.s.fNestedPaging)
2390 {
2391 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2394 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2395 }
2396
2397 /* Use TPR shadowing if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2399 {
2400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2401 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2403 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2404 AssertRCReturn(rc, rc);
2405
2406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2407 /* CR8 writes cause a VM-exit based on TPR threshold. */
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2409 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2410 }
2411 else
2412 {
2413 /*
2414 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2415 * Set this control only for 64-bit guests.
2416 */
2417 if (pVM->hm.s.fAllow64BitGuests)
2418 {
2419 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2421 }
2422 }
2423
2424 /* Use MSR-bitmaps if supported by the CPU. */
2425 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2426 {
2427 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2428
2429 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2430 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2431 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2432 AssertRCReturn(rc, rc);
2433
2434 /*
2435 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2436 * automatically using dedicated fields in the VMCS.
2437 */
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443
2444#if HC_ARCH_BITS == 64
2445 /*
2446 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2447 */
2448 if (pVM->hm.s.fAllow64BitGuests)
2449 {
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2454 }
2455#endif
2456 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2457 }
2458
2459 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2460 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2462
2463 if ((val & zap) != val)
2464 {
2465 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2466 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2467 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2468 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2469 }
2470
2471 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2472 AssertRCReturn(rc, rc);
2473
2474 pVCpu->hm.s.vmx.u32ProcCtls = val;
2475
2476 /*
2477 * Secondary processor-based VM-execution controls.
2478 */
2479 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2480 {
2481 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2482 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2483
2484 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2485 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2486
2487 if (pVM->hm.s.fNestedPaging)
2488 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2489 else
2490 {
2491 /*
2492 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2493 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2494 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2495 */
2496 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2497 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2498 }
2499
2500 if (pVM->hm.s.vmx.fVpid)
2501 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2502
2503 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2504 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2505
2506#if 0
2507 if (pVM->hm.s.fVirtApicRegs)
2508 {
2509 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2511
2512 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2513 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2514 }
2515#endif
2516
2517 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2518 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2519 * done dynamically. */
2520 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2521 {
2522 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2523 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2524 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2525 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2526 AssertRCReturn(rc, rc);
2527 }
2528
2529 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2530 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2531
2532 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2533 && pVM->hm.s.vmx.cPleGapTicks
2534 && pVM->hm.s.vmx.cPleWindowTicks)
2535 {
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2537
2538 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2539 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2540 AssertRCReturn(rc, rc);
2541 }
2542
2543 if ((val & zap) != val)
2544 {
2545 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2546 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2547 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2548 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2549 }
2550
2551 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2552 AssertRCReturn(rc, rc);
2553
2554 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2555 }
2556 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2557 {
2558 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2559 "available\n"));
2560 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2561 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2562 }
2563
2564 return VINF_SUCCESS;
2565}
2566
2567
2568/**
2569 * Sets up miscellaneous (everything other than Pin & Processor-based
2570 * VM-execution) control fields in the VMCS.
2571 *
2572 * @returns VBox status code.
2573 * @param pVM The cross context VM structure.
2574 * @param pVCpu The cross context virtual CPU structure.
2575 */
2576static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2577{
2578 NOREF(pVM);
2579 AssertPtr(pVM);
2580 AssertPtr(pVCpu);
2581
2582 int rc = VERR_GENERAL_FAILURE;
2583
2584 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2585#if 0
2586 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2588 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2589
2590 /*
2591 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2592 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2593 * We thus use the exception bitmap to control it rather than use both.
2594 */
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2597
2598 /** @todo Explore possibility of using IO-bitmaps. */
2599 /* All IO & IOIO instructions cause VM-exits. */
2600 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2601 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2602
2603 /* Initialize the MSR-bitmap area. */
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2606 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2607 AssertRCReturn(rc, rc);
2608#endif
2609
2610 /* Setup MSR auto-load/store area. */
2611 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2612 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2613 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2614 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2615 AssertRCReturn(rc, rc);
2616
2617 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2618 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2620 AssertRCReturn(rc, rc);
2621
2622 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2623 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2624 AssertRCReturn(rc, rc);
2625
2626 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2627#if 0
2628 /* Setup debug controls */
2629 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2630 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2631 AssertRCReturn(rc, rc);
2632#endif
2633
2634 return rc;
2635}
2636
2637
2638/**
2639 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2640 *
2641 * @returns VBox status code.
2642 * @param pVM The cross context VM structure.
2643 * @param pVCpu The cross context virtual CPU structure.
2644 */
2645static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2646{
2647 AssertPtr(pVM);
2648 AssertPtr(pVCpu);
2649
2650 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2651
2652 /** @todo r=ramshankar: Shouldn't setting up \#UD intercepts be handled by
2653 * hmR0VmxLoadGuestXcptIntercepts()? Why do we check it here? */
2654 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2655
2656 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2657 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2658
2659 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2660 and writes, and because recursive #DBs can cause the CPU hang, we must always
2661 intercept #DB. */
2662 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2663
2664 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2665 if (!pVM->hm.s.fNestedPaging)
2666 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2667
2668 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2669 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2670 AssertRCReturn(rc, rc);
2671 return rc;
2672}
2673
2674
2675/**
2676 * Sets up the initial guest-state mask. The guest-state mask is consulted
2677 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2678 * for the nested virtualization case (as it would cause a VM-exit).
2679 *
2680 * @param pVCpu The cross context virtual CPU structure.
2681 */
2682static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2683{
2684 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2685 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x initialization.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700 int rc = hmR0VmxStructsAlloc(pVM);
2701 if (RT_FAILURE(rc))
2702 {
2703 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2704 return rc;
2705 }
2706
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Does per-VM VT-x termination.
2713 *
2714 * @returns VBox status code.
2715 * @param pVM The cross context VM structure.
2716 */
2717VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2718{
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2722 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2723 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2724#endif
2725 hmR0VmxStructsFree(pVM);
2726 return VINF_SUCCESS;
2727}
2728
2729
2730/**
2731 * Sets up the VM for execution under VT-x.
2732 * This function is only called once per-VM during initialization.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM The cross context VM structure.
2736 */
2737VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2738{
2739 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2740 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2741
2742 LogFlowFunc(("pVM=%p\n", pVM));
2743
2744 /*
2745 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2746 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2747 */
2748 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2749 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2750 || !pVM->hm.s.vmx.pRealModeTSS))
2751 {
2752 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2753 return VERR_INTERNAL_ERROR;
2754 }
2755
2756 /* Initialize these always, see hmR3InitFinalizeR0().*/
2757 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2758 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2759
2760 /* Setup the tagged-TLB flush handlers. */
2761 int rc = hmR0VmxSetupTaggedTlb(pVM);
2762 if (RT_FAILURE(rc))
2763 {
2764 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2765 return rc;
2766 }
2767
2768 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2769 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2770#if HC_ARCH_BITS == 64
2771 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2772 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2773 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2774 {
2775 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2776 }
2777#endif
2778
2779 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2780 RTCCUINTREG uHostCR4 = ASMGetCR4();
2781 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2782 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2783
2784 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2785 {
2786 PVMCPU pVCpu = &pVM->aCpus[i];
2787 AssertPtr(pVCpu);
2788 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2789
2790 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2791 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2792
2793 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2794 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2795 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2796
2797 /* Set revision dword at the beginning of the VMCS structure. */
2798 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2799
2800 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2801 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 /* Load this VMCS as the current VMCS. */
2806 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2807 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2808 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2809
2810 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2811 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2812 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2813
2814 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817
2818 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2820 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2821
2822 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2823 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2824 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2825
2826 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2827 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2828 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2829
2830#if HC_ARCH_BITS == 32
2831 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834#endif
2835
2836 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2837 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2838 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2839 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2840
2841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2842
2843 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2844 }
2845
2846 return VINF_SUCCESS;
2847}
2848
2849
2850/**
2851 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2852 * the VMCS.
2853 *
2854 * @returns VBox status code.
2855 * @param pVM The cross context VM structure.
2856 * @param pVCpu The cross context virtual CPU structure.
2857 */
2858DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2859{
2860 NOREF(pVM); NOREF(pVCpu);
2861
2862 RTCCUINTREG uReg = ASMGetCR0();
2863 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2864 AssertRCReturn(rc, rc);
2865
2866 uReg = ASMGetCR3();
2867 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2868 AssertRCReturn(rc, rc);
2869
2870 uReg = ASMGetCR4();
2871 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2872 AssertRCReturn(rc, rc);
2873 return rc;
2874}
2875
2876
2877#if HC_ARCH_BITS == 64
2878/**
2879 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2880 * requirements. See hmR0VmxSaveHostSegmentRegs().
2881 */
2882# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2883 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2884 { \
2885 bool fValidSelector = true; \
2886 if ((selValue) & X86_SEL_LDT) \
2887 { \
2888 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2889 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2890 } \
2891 if (fValidSelector) \
2892 { \
2893 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2894 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2895 } \
2896 (selValue) = 0; \
2897 }
2898#endif
2899
2900
2901/**
2902 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2903 * the host-state area in the VMCS.
2904 *
2905 * @returns VBox status code.
2906 * @param pVM The cross context VM structure.
2907 * @param pVCpu The cross context virtual CPU structure.
2908 */
2909DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2910{
2911 int rc = VERR_INTERNAL_ERROR_5;
2912
2913#if HC_ARCH_BITS == 64
2914 /*
2915 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2916 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2917 *
2918 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2919 * Was observed booting Solaris10u10 32-bit guest.
2920 */
2921 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2922 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2923 {
2924 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2925 pVCpu->idCpu));
2926 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2927 }
2928 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2929#else
2930 RT_NOREF(pVCpu);
2931#endif
2932
2933 /*
2934 * Host DS, ES, FS and GS segment registers.
2935 */
2936#if HC_ARCH_BITS == 64
2937 RTSEL uSelDS = ASMGetDS();
2938 RTSEL uSelES = ASMGetES();
2939 RTSEL uSelFS = ASMGetFS();
2940 RTSEL uSelGS = ASMGetGS();
2941#else
2942 RTSEL uSelDS = 0;
2943 RTSEL uSelES = 0;
2944 RTSEL uSelFS = 0;
2945 RTSEL uSelGS = 0;
2946#endif
2947
2948 /*
2949 * Host CS and SS segment registers.
2950 */
2951 RTSEL uSelCS = ASMGetCS();
2952 RTSEL uSelSS = ASMGetSS();
2953
2954 /*
2955 * Host TR segment register.
2956 */
2957 RTSEL uSelTR = ASMGetTR();
2958
2959#if HC_ARCH_BITS == 64
2960 /*
2961 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2962 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2963 */
2964 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2965 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2966 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2967 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2968# undef VMXLOCAL_ADJUST_HOST_SEG
2969#endif
2970
2971 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2972 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2973 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2974 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2975 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2976 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2977 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2978 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2979 Assert(uSelCS);
2980 Assert(uSelTR);
2981
2982 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2983#if 0
2984 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2985 Assert(uSelSS != 0);
2986#endif
2987
2988 /* Write these host selector fields into the host-state area in the VMCS. */
2989 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2990 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2991#if HC_ARCH_BITS == 64
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2993 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2995 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2996#else
2997 NOREF(uSelDS);
2998 NOREF(uSelES);
2999 NOREF(uSelFS);
3000 NOREF(uSelGS);
3001#endif
3002 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3003 AssertRCReturn(rc, rc);
3004
3005 /*
3006 * Host GDTR and IDTR.
3007 */
3008 RTGDTR Gdtr;
3009 RTIDTR Idtr;
3010 RT_ZERO(Gdtr);
3011 RT_ZERO(Idtr);
3012 ASMGetGDTR(&Gdtr);
3013 ASMGetIDTR(&Idtr);
3014 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3015 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3016 AssertRCReturn(rc, rc);
3017
3018#if HC_ARCH_BITS == 64
3019 /*
3020 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3021 * maximum limit (0xffff) on every VM-exit.
3022 */
3023 if (Gdtr.cbGdt != 0xffff)
3024 {
3025 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3026 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3027 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3028 }
3029
3030 /*
3031 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3032 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3033 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3034 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3035 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3036 * hosts where we are pretty sure it won't cause trouble.
3037 */
3038# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3039 if (Idtr.cbIdt < 0x0fff)
3040# else
3041 if (Idtr.cbIdt != 0xffff)
3042# endif
3043 {
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3045 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3046 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3047 }
3048#endif
3049
3050 /*
3051 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3052 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3053 */
3054 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3055 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3056 VERR_VMX_INVALID_HOST_STATE);
3057
3058 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3059#if HC_ARCH_BITS == 64
3060 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3061
3062 /*
3063 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3064 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3065 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3066 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3067 *
3068 * [1] See Intel spec. 3.5 "System Descriptor Types".
3069 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3070 */
3071 Assert(pDesc->System.u4Type == 11);
3072 if ( pDesc->System.u16LimitLow != 0x67
3073 || pDesc->System.u4LimitHigh)
3074 {
3075 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3076 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3077 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3078 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3079 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3080
3081 /* Store the GDTR here as we need it while restoring TR. */
3082 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3083 }
3084#else
3085 NOREF(pVM);
3086 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3087#endif
3088 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3089 AssertRCReturn(rc, rc);
3090
3091 /*
3092 * Host FS base and GS base.
3093 */
3094#if HC_ARCH_BITS == 64
3095 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3096 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3097 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3098 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3099 AssertRCReturn(rc, rc);
3100
3101 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3102 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3103 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3106#endif
3107 return rc;
3108}
3109
3110
3111/**
3112 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3113 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3114 * the host after every successful VM-exit.
3115 *
3116 * @returns VBox status code.
3117 * @param pVM The cross context VM structure.
3118 * @param pVCpu The cross context virtual CPU structure.
3119 *
3120 * @remarks No-long-jump zone!!!
3121 */
3122DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3123{
3124 NOREF(pVM);
3125
3126 AssertPtr(pVCpu);
3127 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3128
3129 /*
3130 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3131 * rather than swapping them on every VM-entry.
3132 */
3133 hmR0VmxLazySaveHostMsrs(pVCpu);
3134
3135 /*
3136 * Host Sysenter MSRs.
3137 */
3138 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3139#if HC_ARCH_BITS == 32
3140 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3142#else
3143 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3145#endif
3146 AssertRCReturn(rc, rc);
3147
3148 /*
3149 * Host EFER MSR.
3150 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3151 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3152 */
3153 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3154 {
3155 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3156 AssertRCReturn(rc, rc);
3157 }
3158
3159 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3160 * hmR0VmxLoadGuestExitCtls() !! */
3161
3162 return rc;
3163}
3164
3165
3166/**
3167 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3168 *
3169 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3170 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3171 * hmR0VMxLoadGuestEntryCtls().
3172 *
3173 * @returns true if we need to load guest EFER, false otherwise.
3174 * @param pVCpu The cross context virtual CPU structure.
3175 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3176 * out-of-sync. Make sure to update the required fields
3177 * before using them.
3178 *
3179 * @remarks Requires EFER, CR4.
3180 * @remarks No-long-jump zone!!!
3181 */
3182static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3183{
3184#ifdef HMVMX_ALWAYS_SWAP_EFER
3185 return true;
3186#endif
3187
3188#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3189 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3190 if (CPUMIsGuestInLongMode(pVCpu))
3191 return false;
3192#endif
3193
3194 PVM pVM = pVCpu->CTX_SUFF(pVM);
3195 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3196 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3197
3198 /*
3199 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3200 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3201 */
3202 if ( CPUMIsGuestInLongMode(pVCpu)
3203 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3204 {
3205 return true;
3206 }
3207
3208 /*
3209 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3210 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3211 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (pMixedCtx->cr0 & X86_CR0_PG)
3215 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3216 {
3217 /* Assert that host is PAE capable. */
3218 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3219 return true;
3220 }
3221
3222 /** @todo Check the latest Intel spec. for any other bits,
3223 * like SMEP/SMAP? */
3224 return false;
3225}
3226
3227
3228/**
3229 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3230 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3231 * controls".
3232 *
3233 * @returns VBox status code.
3234 * @param pVCpu The cross context virtual CPU structure.
3235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3236 * out-of-sync. Make sure to update the required fields
3237 * before using them.
3238 *
3239 * @remarks Requires EFER.
3240 * @remarks No-long-jump zone!!!
3241 */
3242DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3243{
3244 int rc = VINF_SUCCESS;
3245 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3259 }
3260 else
3261 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 pVCpu->hm.s.vmx.u32EntryCtls = val;
3292 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3293 }
3294 return rc;
3295}
3296
3297
3298/**
3299 * Sets up the VM-exit controls in the VMCS.
3300 *
3301 * @returns VBox status code.
3302 * @param pVCpu The cross context virtual CPU structure.
3303 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3304 * out-of-sync. Make sure to update the required fields
3305 * before using them.
3306 *
3307 * @remarks Requires EFER.
3308 */
3309DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3310{
3311 NOREF(pMixedCtx);
3312
3313 int rc = VINF_SUCCESS;
3314 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3315 {
3316 PVM pVM = pVCpu->CTX_SUFF(pVM);
3317 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3318 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3319
3320 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3321 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3322
3323 /*
3324 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3325 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3326 */
3327#if HC_ARCH_BITS == 64
3328 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3329 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3330#else
3331 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3332 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3333 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3334 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3335 {
3336 /* The switcher returns to long mode, EFER is managed by the switcher. */
3337 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3338 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3339 }
3340 else
3341 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3342#endif
3343
3344 /* If the newer VMCS fields for managing EFER exists, use it. */
3345 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3346 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3347 {
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3349 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3350 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3351 }
3352
3353 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3354 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3355
3356 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3357 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3358 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3359
3360 if ( pVM->hm.s.vmx.fUsePreemptTimer
3361 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3362 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3363
3364 if ((val & zap) != val)
3365 {
3366 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3367 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3368 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3369 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3370 }
3371
3372 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3373 AssertRCReturn(rc, rc);
3374
3375 pVCpu->hm.s.vmx.u32ExitCtls = val;
3376 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3377 }
3378 return rc;
3379}
3380
3381
3382/**
3383 * Sets the TPR threshold in the VMCS.
3384 *
3385 * @returns VBox status code.
3386 * @param pVCpu The cross context virtual CPU structure.
3387 * @param u32TprThreshold The TPR threshold (task-priority class only).
3388 */
3389DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3390{
3391 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3392 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3393 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3394}
3395
3396
3397/**
3398 * Loads the guest APIC and related state.
3399 *
3400 * @returns VBox status code.
3401 * @param pVCpu The cross context virtual CPU structure.
3402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3403 * out-of-sync. Make sure to update the required fields
3404 * before using them.
3405 */
3406DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3407{
3408 NOREF(pMixedCtx);
3409
3410 int rc = VINF_SUCCESS;
3411 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3412 {
3413 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3414 && APICIsEnabled(pVCpu))
3415 {
3416 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3417 {
3418 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3419
3420 bool fPendingIntr = false;
3421 uint8_t u8Tpr = 0;
3422 uint8_t u8PendingIntr = 0;
3423 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3424 AssertRCReturn(rc, rc);
3425
3426 /*
3427 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3428 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3429 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3430 */
3431 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3432 uint32_t u32TprThreshold = 0;
3433 if (fPendingIntr)
3434 {
3435 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3436 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3437 const uint8_t u8TprPriority = u8Tpr >> 4;
3438 if (u8PendingPriority <= u8TprPriority)
3439 u32TprThreshold = u8PendingPriority;
3440 }
3441
3442 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3443 AssertRCReturn(rc, rc);
3444 }
3445
3446#ifndef IEM_VERIFICATION_MODE_FULL
3447 /* Setup the Virtualized APIC accesses. */
3448 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
3449 {
3450 uint64_t u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
3451 if (u64MsrApicBase != pVCpu->hm.s.vmx.u64MsrApicBase)
3452 {
3453 PVM pVM = pVCpu->CTX_SUFF(pVM);
3454 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
3455 RTGCPHYS GCPhysApicBase;
3456 GCPhysApicBase = u64MsrApicBase;
3457 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3458
3459 /* Unalias any existing mapping. */
3460 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
3461 AssertRCReturn(rc, rc);
3462
3463 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
3464 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
3465 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3466 AssertRCReturn(rc, rc);
3467
3468 /* Update VMX's cache of the APIC base. */
3469 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
3470 }
3471 }
3472#endif /* !IEM_VERIFICATION_MODE_FULL */
3473 }
3474 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3475 }
3476
3477 return rc;
3478}
3479
3480
3481/**
3482 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3483 *
3484 * @returns Guest's interruptibility-state.
3485 * @param pVCpu The cross context virtual CPU structure.
3486 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3487 * out-of-sync. Make sure to update the required fields
3488 * before using them.
3489 *
3490 * @remarks No-long-jump zone!!!
3491 */
3492DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3493{
3494 /*
3495 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3496 */
3497 uint32_t uIntrState = 0;
3498 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3499 {
3500 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3501 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3502 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3503 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3504 {
3505 if (pMixedCtx->eflags.Bits.u1IF)
3506 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3507 else
3508 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3509 }
3510 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3511 {
3512 /*
3513 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3514 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3515 */
3516 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3517 }
3518 }
3519
3520 /*
3521 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3522 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3523 * setting this would block host-NMIs and IRET will not clear the blocking.
3524 *
3525 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3526 */
3527 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3528 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3529 {
3530 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3531 }
3532
3533 return uIntrState;
3534}
3535
3536
3537/**
3538 * Loads the guest's interruptibility-state into the guest-state area in the
3539 * VMCS.
3540 *
3541 * @returns VBox status code.
3542 * @param pVCpu The cross context virtual CPU structure.
3543 * @param uIntrState The interruptibility-state to set.
3544 */
3545static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3546{
3547 NOREF(pVCpu);
3548 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3549 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3550 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3551 AssertRC(rc);
3552 return rc;
3553}
3554
3555
3556/**
3557 * Loads the exception intercepts required for guest execution in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu The cross context virtual CPU structure.
3561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3562 * out-of-sync. Make sure to update the required fields
3563 * before using them.
3564 */
3565static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3566{
3567 NOREF(pMixedCtx);
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3570 {
3571 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3572 if (pVCpu->hm.s.fGIMTrapXcptUD)
3573 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3574#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3575 else
3576 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3577#endif
3578
3579 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3580 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3581
3582 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3583 AssertRCReturn(rc, rc);
3584
3585 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3586 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3587 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3588 }
3589 return rc;
3590}
3591
3592
3593/**
3594 * Loads the guest's RIP into the guest-state area in the VMCS.
3595 *
3596 * @returns VBox status code.
3597 * @param pVCpu The cross context virtual CPU structure.
3598 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3599 * out-of-sync. Make sure to update the required fields
3600 * before using them.
3601 *
3602 * @remarks No-long-jump zone!!!
3603 */
3604static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3605{
3606 int rc = VINF_SUCCESS;
3607 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3608 {
3609 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3610 AssertRCReturn(rc, rc);
3611
3612 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3613 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3614 HMCPU_CF_VALUE(pVCpu)));
3615 }
3616 return rc;
3617}
3618
3619
3620/**
3621 * Loads the guest's RSP into the guest-state area in the VMCS.
3622 *
3623 * @returns VBox status code.
3624 * @param pVCpu The cross context virtual CPU structure.
3625 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3626 * out-of-sync. Make sure to update the required fields
3627 * before using them.
3628 *
3629 * @remarks No-long-jump zone!!!
3630 */
3631static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3632{
3633 int rc = VINF_SUCCESS;
3634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3635 {
3636 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3637 AssertRCReturn(rc, rc);
3638
3639 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3640 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3641 }
3642 return rc;
3643}
3644
3645
3646/**
3647 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3648 *
3649 * @returns VBox status code.
3650 * @param pVCpu The cross context virtual CPU structure.
3651 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3652 * out-of-sync. Make sure to update the required fields
3653 * before using them.
3654 *
3655 * @remarks No-long-jump zone!!!
3656 */
3657static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3658{
3659 int rc = VINF_SUCCESS;
3660 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3661 {
3662 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3663 Let us assert it as such and use 32-bit VMWRITE. */
3664 Assert(!(pMixedCtx->rflags.u64 >> 32));
3665 X86EFLAGS Eflags = pMixedCtx->eflags;
3666 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3667 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3668 * These will never be cleared/set, unless some other part of the VMM
3669 * code is buggy - in which case we're better of finding and fixing
3670 * those bugs than hiding them. */
3671 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3672 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3673 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3674 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3675
3676 /*
3677 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3678 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3679 */
3680 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3681 {
3682 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3683 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3684 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3685 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3686 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3687 }
3688
3689 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3690 AssertRCReturn(rc, rc);
3691
3692 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3693 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3694 }
3695 return rc;
3696}
3697
3698
3699/**
3700 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3701 *
3702 * @returns VBox status code.
3703 * @param pVCpu The cross context virtual CPU structure.
3704 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3705 * out-of-sync. Make sure to update the required fields
3706 * before using them.
3707 *
3708 * @remarks No-long-jump zone!!!
3709 */
3710DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3711{
3712 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3713 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3714 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3715 AssertRCReturn(rc, rc);
3716 return rc;
3717}
3718
3719
3720/**
3721 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3722 * CR0 is partially shared with the host and we have to consider the FPU bits.
3723 *
3724 * @returns VBox status code.
3725 * @param pVCpu The cross context virtual CPU structure.
3726 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3727 * out-of-sync. Make sure to update the required fields
3728 * before using them.
3729 *
3730 * @remarks No-long-jump zone!!!
3731 */
3732static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3733{
3734 /*
3735 * Guest CR0.
3736 * Guest FPU.
3737 */
3738 int rc = VINF_SUCCESS;
3739 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3740 {
3741 Assert(!(pMixedCtx->cr0 >> 32));
3742 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3743 PVM pVM = pVCpu->CTX_SUFF(pVM);
3744
3745 /* The guest's view (read access) of its CR0 is unblemished. */
3746 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3747 AssertRCReturn(rc, rc);
3748 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3749
3750 /* Setup VT-x's view of the guest CR0. */
3751 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3752 if (pVM->hm.s.fNestedPaging)
3753 {
3754 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3755 {
3756 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3757 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3758 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3759 }
3760 else
3761 {
3762 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3763 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3764 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3765 }
3766
3767 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3768 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3769 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3770
3771 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3772 AssertRCReturn(rc, rc);
3773 }
3774 else
3775 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3776
3777 /*
3778 * Guest FPU bits.
3779 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3780 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3781 */
3782 u32GuestCR0 |= X86_CR0_NE;
3783 bool fInterceptNM = false;
3784 if (CPUMIsGuestFPUStateActive(pVCpu))
3785 {
3786 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3787 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3788 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3789 }
3790 else
3791 {
3792 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3793 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3794 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3795 }
3796
3797 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3798 bool fInterceptMF = false;
3799 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3800 fInterceptMF = true;
3801
3802 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3803 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3804 {
3805 Assert(PDMVmmDevHeapIsEnabled(pVM));
3806 Assert(pVM->hm.s.vmx.pRealModeTSS);
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3808 fInterceptNM = true;
3809 fInterceptMF = true;
3810 }
3811 else
3812 {
3813 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3814 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3815 }
3816 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3817
3818 if (fInterceptNM)
3819 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3820 else
3821 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3822
3823 if (fInterceptMF)
3824 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3825 else
3826 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3827
3828 /* Additional intercepts for debugging, define these yourself explicitly. */
3829#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3830 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3831 | RT_BIT(X86_XCPT_BP)
3832 | RT_BIT(X86_XCPT_DE)
3833 | RT_BIT(X86_XCPT_NM)
3834 | RT_BIT(X86_XCPT_TS)
3835 | RT_BIT(X86_XCPT_UD)
3836 | RT_BIT(X86_XCPT_NP)
3837 | RT_BIT(X86_XCPT_SS)
3838 | RT_BIT(X86_XCPT_GP)
3839 | RT_BIT(X86_XCPT_PF)
3840 | RT_BIT(X86_XCPT_MF)
3841 ;
3842#elif defined(HMVMX_ALWAYS_TRAP_PF)
3843 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3844#endif
3845
3846 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3847
3848 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3849 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3850 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3851 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3852 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3853 else
3854 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3855
3856 u32GuestCR0 |= uSetCR0;
3857 u32GuestCR0 &= uZapCR0;
3858 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3859
3860 /* Write VT-x's view of the guest CR0 into the VMCS. */
3861 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3862 AssertRCReturn(rc, rc);
3863 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3864 uZapCR0));
3865
3866 /*
3867 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3868 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3869 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3870 */
3871 uint32_t u32CR0Mask = 0;
3872 u32CR0Mask = X86_CR0_PE
3873 | X86_CR0_NE
3874 | X86_CR0_WP
3875 | X86_CR0_PG
3876 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3877 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3878 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3879
3880 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3881 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3882 * and @bugref{6944}. */
3883#if 0
3884 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3885 u32CR0Mask &= ~X86_CR0_PE;
3886#endif
3887 if (pVM->hm.s.fNestedPaging)
3888 u32CR0Mask &= ~X86_CR0_WP;
3889
3890 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3891 if (fInterceptNM)
3892 {
3893 u32CR0Mask |= X86_CR0_TS
3894 | X86_CR0_MP;
3895 }
3896
3897 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3898 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3899 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3900 AssertRCReturn(rc, rc);
3901 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3902
3903 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3904 }
3905 return rc;
3906}
3907
3908
3909/**
3910 * Loads the guest control registers (CR3, CR4) into the guest-state area
3911 * in the VMCS.
3912 *
3913 * @returns VBox strict status code.
3914 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3915 * without unrestricted guest access and the VMMDev is not presently
3916 * mapped (e.g. EFI32).
3917 *
3918 * @param pVCpu The cross context virtual CPU structure.
3919 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3920 * out-of-sync. Make sure to update the required fields
3921 * before using them.
3922 *
3923 * @remarks No-long-jump zone!!!
3924 */
3925static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3926{
3927 int rc = VINF_SUCCESS;
3928 PVM pVM = pVCpu->CTX_SUFF(pVM);
3929
3930 /*
3931 * Guest CR2.
3932 * It's always loaded in the assembler code. Nothing to do here.
3933 */
3934
3935 /*
3936 * Guest CR3.
3937 */
3938 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3939 {
3940 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3941 if (pVM->hm.s.fNestedPaging)
3942 {
3943 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3944
3945 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3946 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3947 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3948 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3949
3950 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3951 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3952 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3953
3954 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3955 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3956 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3957 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3958 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3959 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3960 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3961
3962 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3963 AssertRCReturn(rc, rc);
3964 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3965
3966 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3967 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3968 {
3969 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3970 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3971 {
3972 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3973 AssertRCReturn(rc, rc);
3974 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3975 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3976 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3977 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3978 AssertRCReturn(rc, rc);
3979 }
3980
3981 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3982 have Unrestricted Execution to handle the guest when it's not using paging. */
3983 GCPhysGuestCR3 = pMixedCtx->cr3;
3984 }
3985 else
3986 {
3987 /*
3988 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3989 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3990 * EPT takes care of translating it to host-physical addresses.
3991 */
3992 RTGCPHYS GCPhys;
3993 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3994
3995 /* We obtain it here every time as the guest could have relocated this PCI region. */
3996 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3997 if (RT_SUCCESS(rc))
3998 { /* likely */ }
3999 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4000 {
4001 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4002 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4003 }
4004 else
4005 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4006
4007 GCPhysGuestCR3 = GCPhys;
4008 }
4009
4010 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4011 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4012 }
4013 else
4014 {
4015 /* Non-nested paging case, just use the hypervisor's CR3. */
4016 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4017
4018 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4019 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4020 }
4021 AssertRCReturn(rc, rc);
4022
4023 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4024 }
4025
4026 /*
4027 * Guest CR4.
4028 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4029 */
4030 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4031 {
4032 Assert(!(pMixedCtx->cr4 >> 32));
4033 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4034
4035 /* The guest's view of its CR4 is unblemished. */
4036 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4037 AssertRCReturn(rc, rc);
4038 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4039
4040 /* Setup VT-x's view of the guest CR4. */
4041 /*
4042 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4043 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4044 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4045 */
4046 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4047 {
4048 Assert(pVM->hm.s.vmx.pRealModeTSS);
4049 Assert(PDMVmmDevHeapIsEnabled(pVM));
4050 u32GuestCR4 &= ~X86_CR4_VME;
4051 }
4052
4053 if (pVM->hm.s.fNestedPaging)
4054 {
4055 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4056 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4057 {
4058 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4059 u32GuestCR4 |= X86_CR4_PSE;
4060 /* Our identity mapping is a 32-bit page directory. */
4061 u32GuestCR4 &= ~X86_CR4_PAE;
4062 }
4063 /* else use guest CR4.*/
4064 }
4065 else
4066 {
4067 /*
4068 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4069 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4070 */
4071 switch (pVCpu->hm.s.enmShadowMode)
4072 {
4073 case PGMMODE_REAL: /* Real-mode. */
4074 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4075 case PGMMODE_32_BIT: /* 32-bit paging. */
4076 {
4077 u32GuestCR4 &= ~X86_CR4_PAE;
4078 break;
4079 }
4080
4081 case PGMMODE_PAE: /* PAE paging. */
4082 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4083 {
4084 u32GuestCR4 |= X86_CR4_PAE;
4085 break;
4086 }
4087
4088 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4089 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4090#ifdef VBOX_ENABLE_64_BITS_GUESTS
4091 break;
4092#endif
4093 default:
4094 AssertFailed();
4095 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4096 }
4097 }
4098
4099 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4100 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4101 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4102 u32GuestCR4 |= uSetCR4;
4103 u32GuestCR4 &= uZapCR4;
4104
4105 /* Write VT-x's view of the guest CR4 into the VMCS. */
4106 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4107 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4108 AssertRCReturn(rc, rc);
4109
4110 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4111 uint32_t u32CR4Mask = X86_CR4_VME
4112 | X86_CR4_PAE
4113 | X86_CR4_PGE
4114 | X86_CR4_PSE
4115 | X86_CR4_VMXE;
4116 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4117 u32CR4Mask |= X86_CR4_OSXSAVE;
4118 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4119 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4120 AssertRCReturn(rc, rc);
4121
4122 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4123 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4124
4125 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4126 }
4127 return rc;
4128}
4129
4130
4131/**
4132 * Loads the guest debug registers into the guest-state area in the VMCS.
4133 *
4134 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4135 *
4136 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4137 *
4138 * @returns VBox status code.
4139 * @param pVCpu The cross context virtual CPU structure.
4140 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4141 * out-of-sync. Make sure to update the required fields
4142 * before using them.
4143 *
4144 * @remarks No-long-jump zone!!!
4145 */
4146static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4147{
4148 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4149 return VINF_SUCCESS;
4150
4151#ifdef VBOX_STRICT
4152 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4153 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4154 {
4155 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4156 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4157 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4158 }
4159#endif
4160
4161 int rc;
4162 PVM pVM = pVCpu->CTX_SUFF(pVM);
4163 bool fSteppingDB = false;
4164 bool fInterceptMovDRx = false;
4165 if (pVCpu->hm.s.fSingleInstruction)
4166 {
4167 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4168 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4169 {
4170 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4171 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4172 AssertRCReturn(rc, rc);
4173 Assert(fSteppingDB == false);
4174 }
4175 else
4176 {
4177 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4178 pVCpu->hm.s.fClearTrapFlag = true;
4179 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4180 fSteppingDB = true;
4181 }
4182 }
4183
4184 if ( fSteppingDB
4185 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4186 {
4187 /*
4188 * Use the combined guest and host DRx values found in the hypervisor
4189 * register set because the debugger has breakpoints active or someone
4190 * is single stepping on the host side without a monitor trap flag.
4191 *
4192 * Note! DBGF expects a clean DR6 state before executing guest code.
4193 */
4194#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4195 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4196 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4197 {
4198 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4199 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4200 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4201 }
4202 else
4203#endif
4204 if (!CPUMIsHyperDebugStateActive(pVCpu))
4205 {
4206 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4207 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4208 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4209 }
4210
4211 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4212 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4213 AssertRCReturn(rc, rc);
4214
4215 pVCpu->hm.s.fUsingHyperDR7 = true;
4216 fInterceptMovDRx = true;
4217 }
4218 else
4219 {
4220 /*
4221 * If the guest has enabled debug registers, we need to load them prior to
4222 * executing guest code so they'll trigger at the right time.
4223 */
4224 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4225 {
4226#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4227 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4228 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4229 {
4230 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4231 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4232 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4233 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4234 }
4235 else
4236#endif
4237 if (!CPUMIsGuestDebugStateActive(pVCpu))
4238 {
4239 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4240 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4241 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4242 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4243 }
4244 Assert(!fInterceptMovDRx);
4245 }
4246 /*
4247 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4248 * must intercept #DB in order to maintain a correct DR6 guest value, and
4249 * because we need to intercept it to prevent nested #DBs from hanging the
4250 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4251 */
4252#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4253 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4254 && !CPUMIsGuestDebugStateActive(pVCpu))
4255#else
4256 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4257#endif
4258 {
4259 fInterceptMovDRx = true;
4260 }
4261
4262 /* Update guest DR7. */
4263 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4264 AssertRCReturn(rc, rc);
4265
4266 pVCpu->hm.s.fUsingHyperDR7 = false;
4267 }
4268
4269 /*
4270 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4271 */
4272 if (fInterceptMovDRx)
4273 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4274 else
4275 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4276 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4277 AssertRCReturn(rc, rc);
4278
4279 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4280 return VINF_SUCCESS;
4281}
4282
4283
4284#ifdef VBOX_STRICT
4285/**
4286 * Strict function to validate segment registers.
4287 *
4288 * @remarks ASSUMES CR0 is up to date.
4289 */
4290static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4291{
4292 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4293 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4294 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4295 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4296 && ( !CPUMIsGuestInRealModeEx(pCtx)
4297 && !CPUMIsGuestInV86ModeEx(pCtx)))
4298 {
4299 /* Protected mode checks */
4300 /* CS */
4301 Assert(pCtx->cs.Attr.n.u1Present);
4302 Assert(!(pCtx->cs.Attr.u & 0xf00));
4303 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4304 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4305 || !(pCtx->cs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4307 || (pCtx->cs.Attr.n.u1Granularity));
4308 /* CS cannot be loaded with NULL in protected mode. */
4309 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4310 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4311 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4312 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4313 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4314 else
4315 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4316 /* SS */
4317 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4318 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4319 if ( !(pCtx->cr0 & X86_CR0_PE)
4320 || pCtx->cs.Attr.n.u4Type == 3)
4321 {
4322 Assert(!pCtx->ss.Attr.n.u2Dpl);
4323 }
4324 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4325 {
4326 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4327 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4328 Assert(pCtx->ss.Attr.n.u1Present);
4329 Assert(!(pCtx->ss.Attr.u & 0xf00));
4330 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4331 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4332 || !(pCtx->ss.Attr.n.u1Granularity));
4333 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4334 || (pCtx->ss.Attr.n.u1Granularity));
4335 }
4336 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4337 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->ds.Attr.n.u1Present);
4341 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->ds.Attr.u & 0xf00));
4343 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->ds.Attr.n.u1Granularity));
4346 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4347 || (pCtx->ds.Attr.n.u1Granularity));
4348 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4352 {
4353 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4354 Assert(pCtx->es.Attr.n.u1Present);
4355 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4356 Assert(!(pCtx->es.Attr.u & 0xf00));
4357 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4358 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4359 || !(pCtx->es.Attr.n.u1Granularity));
4360 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4361 || (pCtx->es.Attr.n.u1Granularity));
4362 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4363 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4364 }
4365 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4366 {
4367 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4368 Assert(pCtx->fs.Attr.n.u1Present);
4369 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4370 Assert(!(pCtx->fs.Attr.u & 0xf00));
4371 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4372 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4373 || !(pCtx->fs.Attr.n.u1Granularity));
4374 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4375 || (pCtx->fs.Attr.n.u1Granularity));
4376 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4377 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4378 }
4379 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4380 {
4381 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4382 Assert(pCtx->gs.Attr.n.u1Present);
4383 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4384 Assert(!(pCtx->gs.Attr.u & 0xf00));
4385 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4386 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4387 || !(pCtx->gs.Attr.n.u1Granularity));
4388 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4389 || (pCtx->gs.Attr.n.u1Granularity));
4390 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4391 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4392 }
4393 /* 64-bit capable CPUs. */
4394# if HC_ARCH_BITS == 64
4395 Assert(!(pCtx->cs.u64Base >> 32));
4396 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4397 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4398 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4399# endif
4400 }
4401 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4402 || ( CPUMIsGuestInRealModeEx(pCtx)
4403 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4404 {
4405 /* Real and v86 mode checks. */
4406 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4407 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4408 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4409 {
4410 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4411 }
4412 else
4413 {
4414 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4415 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4416 }
4417
4418 /* CS */
4419 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4420 Assert(pCtx->cs.u32Limit == 0xffff);
4421 Assert(u32CSAttr == 0xf3);
4422 /* SS */
4423 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4424 Assert(pCtx->ss.u32Limit == 0xffff);
4425 Assert(u32SSAttr == 0xf3);
4426 /* DS */
4427 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4428 Assert(pCtx->ds.u32Limit == 0xffff);
4429 Assert(u32DSAttr == 0xf3);
4430 /* ES */
4431 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4432 Assert(pCtx->es.u32Limit == 0xffff);
4433 Assert(u32ESAttr == 0xf3);
4434 /* FS */
4435 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4436 Assert(pCtx->fs.u32Limit == 0xffff);
4437 Assert(u32FSAttr == 0xf3);
4438 /* GS */
4439 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4440 Assert(pCtx->gs.u32Limit == 0xffff);
4441 Assert(u32GSAttr == 0xf3);
4442 /* 64-bit capable CPUs. */
4443# if HC_ARCH_BITS == 64
4444 Assert(!(pCtx->cs.u64Base >> 32));
4445 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4446 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4447 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4448# endif
4449 }
4450}
4451#endif /* VBOX_STRICT */
4452
4453
4454/**
4455 * Writes a guest segment register into the guest-state area in the VMCS.
4456 *
4457 * @returns VBox status code.
4458 * @param pVCpu The cross context virtual CPU structure.
4459 * @param idxSel Index of the selector in the VMCS.
4460 * @param idxLimit Index of the segment limit in the VMCS.
4461 * @param idxBase Index of the segment base in the VMCS.
4462 * @param idxAccess Index of the access rights of the segment in the VMCS.
4463 * @param pSelReg Pointer to the segment selector.
4464 *
4465 * @remarks No-long-jump zone!!!
4466 */
4467static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4468 uint32_t idxAccess, PCPUMSELREG pSelReg)
4469{
4470 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4471 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4472 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4473 AssertRCReturn(rc, rc);
4474
4475 uint32_t u32Access = pSelReg->Attr.u;
4476 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4477 {
4478 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4479 u32Access = 0xf3;
4480 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4481 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4482 }
4483 else
4484 {
4485 /*
4486 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4487 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4488 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4489 * loaded in protected-mode have their attribute as 0.
4490 */
4491 if (!u32Access)
4492 u32Access = X86DESCATTR_UNUSABLE;
4493 }
4494
4495 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4496 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4497 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4498
4499 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4500 AssertRCReturn(rc, rc);
4501 return rc;
4502}
4503
4504
4505/**
4506 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4507 * into the guest-state area in the VMCS.
4508 *
4509 * @returns VBox status code.
4510 * @param pVCpu The cross context virtual CPU structure.
4511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4512 * out-of-sync. Make sure to update the required fields
4513 * before using them.
4514 *
4515 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4516 * @remarks No-long-jump zone!!!
4517 */
4518static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4519{
4520 int rc = VERR_INTERNAL_ERROR_5;
4521 PVM pVM = pVCpu->CTX_SUFF(pVM);
4522
4523 /*
4524 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4525 */
4526 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4527 {
4528 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4530 {
4531 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4532 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4533 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4534 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4535 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4536 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4537 }
4538
4539#ifdef VBOX_WITH_REM
4540 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4541 {
4542 Assert(pVM->hm.s.vmx.pRealModeTSS);
4543 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4544 if ( pVCpu->hm.s.vmx.fWasInRealMode
4545 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4546 {
4547 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4548 in real-mode (e.g. OpenBSD 4.0) */
4549 REMFlushTBs(pVM);
4550 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4551 pVCpu->hm.s.vmx.fWasInRealMode = false;
4552 }
4553 }
4554#endif
4555 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4556 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4557 AssertRCReturn(rc, rc);
4558 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4559 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4560 AssertRCReturn(rc, rc);
4561 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4562 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4563 AssertRCReturn(rc, rc);
4564 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4565 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4566 AssertRCReturn(rc, rc);
4567 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4568 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4569 AssertRCReturn(rc, rc);
4570 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4571 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4572 AssertRCReturn(rc, rc);
4573
4574#ifdef VBOX_STRICT
4575 /* Validate. */
4576 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4577#endif
4578
4579 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4580 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4581 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4582 }
4583
4584 /*
4585 * Guest TR.
4586 */
4587 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4588 {
4589 /*
4590 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4591 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4592 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4593 */
4594 uint16_t u16Sel = 0;
4595 uint32_t u32Limit = 0;
4596 uint64_t u64Base = 0;
4597 uint32_t u32AccessRights = 0;
4598
4599 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4600 {
4601 u16Sel = pMixedCtx->tr.Sel;
4602 u32Limit = pMixedCtx->tr.u32Limit;
4603 u64Base = pMixedCtx->tr.u64Base;
4604 u32AccessRights = pMixedCtx->tr.Attr.u;
4605 }
4606 else
4607 {
4608 Assert(pVM->hm.s.vmx.pRealModeTSS);
4609 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4610
4611 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4612 RTGCPHYS GCPhys;
4613 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4614 AssertRCReturn(rc, rc);
4615
4616 X86DESCATTR DescAttr;
4617 DescAttr.u = 0;
4618 DescAttr.n.u1Present = 1;
4619 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4620
4621 u16Sel = 0;
4622 u32Limit = HM_VTX_TSS_SIZE;
4623 u64Base = GCPhys; /* in real-mode phys = virt. */
4624 u32AccessRights = DescAttr.u;
4625 }
4626
4627 /* Validate. */
4628 Assert(!(u16Sel & RT_BIT(2)));
4629 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4630 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4631 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4632 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4633 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4634 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4635 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4636 Assert( (u32Limit & 0xfff) == 0xfff
4637 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4638 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4639 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4640
4641 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4642 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4643 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4645 AssertRCReturn(rc, rc);
4646
4647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4648 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4649 }
4650
4651 /*
4652 * Guest GDTR.
4653 */
4654 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4655 {
4656 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4657 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4658 AssertRCReturn(rc, rc);
4659
4660 /* Validate. */
4661 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4662
4663 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4664 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4665 }
4666
4667 /*
4668 * Guest LDTR.
4669 */
4670 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4671 {
4672 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4673 uint32_t u32Access = 0;
4674 if (!pMixedCtx->ldtr.Attr.u)
4675 u32Access = X86DESCATTR_UNUSABLE;
4676 else
4677 u32Access = pMixedCtx->ldtr.Attr.u;
4678
4679 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4680 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4681 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4682 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4683 AssertRCReturn(rc, rc);
4684
4685 /* Validate. */
4686 if (!(u32Access & X86DESCATTR_UNUSABLE))
4687 {
4688 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4689 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4690 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4691 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4692 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4693 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4694 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4695 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4696 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4697 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4698 }
4699
4700 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4701 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4702 }
4703
4704 /*
4705 * Guest IDTR.
4706 */
4707 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4708 {
4709 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4710 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4711 AssertRCReturn(rc, rc);
4712
4713 /* Validate. */
4714 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4715
4716 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4717 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4718 }
4719
4720 return VINF_SUCCESS;
4721}
4722
4723
4724/**
4725 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4726 * areas.
4727 *
4728 * These MSRs will automatically be loaded to the host CPU on every successful
4729 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4730 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4731 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4732 *
4733 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4734 *
4735 * @returns VBox status code.
4736 * @param pVCpu The cross context virtual CPU structure.
4737 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4738 * out-of-sync. Make sure to update the required fields
4739 * before using them.
4740 *
4741 * @remarks No-long-jump zone!!!
4742 */
4743static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744{
4745 AssertPtr(pVCpu);
4746 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4747
4748 /*
4749 * MSRs that we use the auto-load/store MSR area in the VMCS.
4750 */
4751 PVM pVM = pVCpu->CTX_SUFF(pVM);
4752 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4753 {
4754 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4755#if HC_ARCH_BITS == 32
4756 if (pVM->hm.s.fAllow64BitGuests)
4757 {
4758 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4759 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4760 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4761 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4762 AssertRCReturn(rc, rc);
4763# ifdef LOG_ENABLED
4764 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4765 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4766 {
4767 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4768 pMsr->u64Value));
4769 }
4770# endif
4771 }
4772#endif
4773 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4774 }
4775
4776 /*
4777 * Guest Sysenter MSRs.
4778 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4779 * VM-exits on WRMSRs for these MSRs.
4780 */
4781 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4782 {
4783 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4784 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4785 }
4786
4787 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4788 {
4789 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4790 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4791 }
4792
4793 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4794 {
4795 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4796 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4797 }
4798
4799 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4800 {
4801 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4802 {
4803 /*
4804 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4805 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4806 */
4807 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4808 {
4809 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4810 AssertRCReturn(rc,rc);
4811 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4812 }
4813 else
4814 {
4815 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4816 NULL /* pfAddedAndUpdated */);
4817 AssertRCReturn(rc, rc);
4818
4819 /* We need to intercept reads too, see @bugref{7386#c16}. */
4820 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4821 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4822 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4823 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4824 }
4825 }
4826 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4827 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4828 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4829 }
4830
4831 return VINF_SUCCESS;
4832}
4833
4834
4835/**
4836 * Loads the guest activity state into the guest-state area in the VMCS.
4837 *
4838 * @returns VBox status code.
4839 * @param pVCpu The cross context virtual CPU structure.
4840 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4841 * out-of-sync. Make sure to update the required fields
4842 * before using them.
4843 *
4844 * @remarks No-long-jump zone!!!
4845 */
4846static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4847{
4848 NOREF(pMixedCtx);
4849 /** @todo See if we can make use of other states, e.g.
4850 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4851 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4852 {
4853 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4854 AssertRCReturn(rc, rc);
4855
4856 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4857 }
4858 return VINF_SUCCESS;
4859}
4860
4861
4862#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4863/**
4864 * Check if guest state allows safe use of 32-bit switcher again.
4865 *
4866 * Segment bases and protected mode structures must be 32-bit addressable
4867 * because the 32-bit switcher will ignore high dword when writing these VMCS
4868 * fields. See @bugref{8432} for details.
4869 *
4870 * @returns true if safe, false if must continue to use the 64-bit switcher.
4871 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4872 * out-of-sync. Make sure to update the required fields
4873 * before using them.
4874 *
4875 * @remarks No-long-jump zone!!!
4876 */
4877static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4878{
4879 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4880 return false;
4881 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4882 return false;
4883 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4884 return false;
4885 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4886 return false;
4887 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4888 return false;
4889 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4890 return false;
4891 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4892 return false;
4893 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4894 return false;
4895 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4896 return false;
4897 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4898 return false;
4899 /* All good, bases are 32-bit. */
4900 return true;
4901}
4902#endif
4903
4904
4905/**
4906 * Sets up the appropriate function to run guest code.
4907 *
4908 * @returns VBox status code.
4909 * @param pVCpu The cross context virtual CPU structure.
4910 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4911 * out-of-sync. Make sure to update the required fields
4912 * before using them.
4913 *
4914 * @remarks No-long-jump zone!!!
4915 */
4916static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4917{
4918 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4919 {
4920#ifndef VBOX_ENABLE_64_BITS_GUESTS
4921 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4922#endif
4923 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4924#if HC_ARCH_BITS == 32
4925 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4926 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4927 {
4928 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4929 {
4930 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4931 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4932 | HM_CHANGED_VMX_ENTRY_CTLS
4933 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4934 }
4935 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4936
4937 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4938 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4939 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4940 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4941 }
4942#else
4943 /* 64-bit host. */
4944 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4945#endif
4946 }
4947 else
4948 {
4949 /* Guest is not in long mode, use the 32-bit handler. */
4950#if HC_ARCH_BITS == 32
4951 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4952 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4953 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4954 {
4955 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4956 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4957 | HM_CHANGED_VMX_ENTRY_CTLS
4958 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4959 }
4960# ifdef VBOX_ENABLE_64_BITS_GUESTS
4961 /*
4962 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4963 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4964 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4965 * the much faster 32-bit switcher again.
4966 */
4967 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4968 {
4969 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4970 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4971 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972 }
4973 else
4974 {
4975 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4976 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4977 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4978 {
4979 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4980 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4981 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4982 | HM_CHANGED_VMX_ENTRY_CTLS
4983 | HM_CHANGED_VMX_EXIT_CTLS
4984 | HM_CHANGED_HOST_CONTEXT);
4985 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
4986 }
4987 }
4988# else
4989 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4990# endif
4991#else
4992 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4993#endif
4994 }
4995 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4996 return VINF_SUCCESS;
4997}
4998
4999
5000/**
5001 * Wrapper for running the guest code in VT-x.
5002 *
5003 * @returns VBox status code, no informational status codes.
5004 * @param pVM The cross context VM structure.
5005 * @param pVCpu The cross context virtual CPU structure.
5006 * @param pCtx Pointer to the guest-CPU context.
5007 *
5008 * @remarks No-long-jump zone!!!
5009 */
5010DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5011{
5012 /*
5013 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5014 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5015 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5016 */
5017 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5018 /** @todo Add stats for resume vs launch. */
5019#ifdef VBOX_WITH_KERNEL_USING_XMM
5020 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5021#else
5022 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5023#endif
5024 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5025 return rc;
5026}
5027
5028
5029/**
5030 * Reports world-switch error and dumps some useful debug info.
5031 *
5032 * @param pVM The cross context VM structure.
5033 * @param pVCpu The cross context virtual CPU structure.
5034 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5035 * @param pCtx Pointer to the guest-CPU context.
5036 * @param pVmxTransient Pointer to the VMX transient structure (only
5037 * exitReason updated).
5038 */
5039static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5040{
5041 Assert(pVM);
5042 Assert(pVCpu);
5043 Assert(pCtx);
5044 Assert(pVmxTransient);
5045 HMVMX_ASSERT_PREEMPT_SAFE();
5046
5047 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5048 switch (rcVMRun)
5049 {
5050 case VERR_VMX_INVALID_VMXON_PTR:
5051 AssertFailed();
5052 break;
5053 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5054 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5055 {
5056 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5057 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5058 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5059 AssertRC(rc);
5060
5061 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5062 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5063 Cannot do it here as we may have been long preempted. */
5064
5065#ifdef VBOX_STRICT
5066 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5067 pVmxTransient->uExitReason));
5068 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5069 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5070 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5071 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5072 else
5073 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5074 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5075 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5076
5077 /* VMX control bits. */
5078 uint32_t u32Val;
5079 uint64_t u64Val;
5080 RTHCUINTREG uHCReg;
5081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5085 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5086 {
5087 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5088 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5089 }
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5107 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5113 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5116 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5117 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5119 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5121 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5122 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5123 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5124 if (pVM->hm.s.fNestedPaging)
5125 {
5126 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5127 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5128 }
5129
5130 /* Guest bits. */
5131 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5132 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5133 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5134 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5135 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5136 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5137 if (pVM->hm.s.vmx.fVpid)
5138 {
5139 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5140 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5141 }
5142
5143 /* Host bits. */
5144 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5145 Log4(("Host CR0 %#RHr\n", uHCReg));
5146 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5147 Log4(("Host CR3 %#RHr\n", uHCReg));
5148 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5149 Log4(("Host CR4 %#RHr\n", uHCReg));
5150
5151 RTGDTR HostGdtr;
5152 PCX86DESCHC pDesc;
5153 ASMGetGDTR(&HostGdtr);
5154 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5155 Log4(("Host CS %#08x\n", u32Val));
5156 if (u32Val < HostGdtr.cbGdt)
5157 {
5158 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5159 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5160 }
5161
5162 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5163 Log4(("Host DS %#08x\n", u32Val));
5164 if (u32Val < HostGdtr.cbGdt)
5165 {
5166 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5167 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5168 }
5169
5170 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5171 Log4(("Host ES %#08x\n", u32Val));
5172 if (u32Val < HostGdtr.cbGdt)
5173 {
5174 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5175 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5176 }
5177
5178 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5179 Log4(("Host FS %#08x\n", u32Val));
5180 if (u32Val < HostGdtr.cbGdt)
5181 {
5182 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5183 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5184 }
5185
5186 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5187 Log4(("Host GS %#08x\n", u32Val));
5188 if (u32Val < HostGdtr.cbGdt)
5189 {
5190 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5191 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5192 }
5193
5194 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5195 Log4(("Host SS %#08x\n", u32Val));
5196 if (u32Val < HostGdtr.cbGdt)
5197 {
5198 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5199 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5200 }
5201
5202 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5203 Log4(("Host TR %#08x\n", u32Val));
5204 if (u32Val < HostGdtr.cbGdt)
5205 {
5206 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5207 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5208 }
5209
5210 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5211 Log4(("Host TR Base %#RHv\n", uHCReg));
5212 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5213 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5215 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5216 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5217 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5218 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5219 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5220 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5221 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5222 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5223 Log4(("Host RSP %#RHv\n", uHCReg));
5224 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5225 Log4(("Host RIP %#RHv\n", uHCReg));
5226# if HC_ARCH_BITS == 64
5227 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5228 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5229 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5230 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5231 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5232 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5233# endif
5234#endif /* VBOX_STRICT */
5235 break;
5236 }
5237
5238 default:
5239 /* Impossible */
5240 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5241 break;
5242 }
5243 NOREF(pVM); NOREF(pCtx);
5244}
5245
5246
5247#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5248#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5249# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5250#endif
5251#ifdef VBOX_STRICT
5252static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5253{
5254 switch (idxField)
5255 {
5256 case VMX_VMCS_GUEST_RIP:
5257 case VMX_VMCS_GUEST_RSP:
5258 case VMX_VMCS_GUEST_SYSENTER_EIP:
5259 case VMX_VMCS_GUEST_SYSENTER_ESP:
5260 case VMX_VMCS_GUEST_GDTR_BASE:
5261 case VMX_VMCS_GUEST_IDTR_BASE:
5262 case VMX_VMCS_GUEST_CS_BASE:
5263 case VMX_VMCS_GUEST_DS_BASE:
5264 case VMX_VMCS_GUEST_ES_BASE:
5265 case VMX_VMCS_GUEST_FS_BASE:
5266 case VMX_VMCS_GUEST_GS_BASE:
5267 case VMX_VMCS_GUEST_SS_BASE:
5268 case VMX_VMCS_GUEST_LDTR_BASE:
5269 case VMX_VMCS_GUEST_TR_BASE:
5270 case VMX_VMCS_GUEST_CR3:
5271 return true;
5272 }
5273 return false;
5274}
5275
5276static bool hmR0VmxIsValidReadField(uint32_t idxField)
5277{
5278 switch (idxField)
5279 {
5280 /* Read-only fields. */
5281 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5282 return true;
5283 }
5284 /* Remaining readable fields should also be writable. */
5285 return hmR0VmxIsValidWriteField(idxField);
5286}
5287#endif /* VBOX_STRICT */
5288
5289
5290/**
5291 * Executes the specified handler in 64-bit mode.
5292 *
5293 * @returns VBox status code (no informational status codes).
5294 * @param pVM The cross context VM structure.
5295 * @param pVCpu The cross context virtual CPU structure.
5296 * @param pCtx Pointer to the guest CPU context.
5297 * @param enmOp The operation to perform.
5298 * @param cParams Number of parameters.
5299 * @param paParam Array of 32-bit parameters.
5300 */
5301VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5302 uint32_t cParams, uint32_t *paParam)
5303{
5304 NOREF(pCtx);
5305
5306 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5307 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5308 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5309 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5310
5311#ifdef VBOX_STRICT
5312 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5313 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5314
5315 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5316 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5317#endif
5318
5319 /* Disable interrupts. */
5320 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5321
5322#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5323 RTCPUID idHostCpu = RTMpCpuId();
5324 CPUMR0SetLApic(pVCpu, idHostCpu);
5325#endif
5326
5327 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5328 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5329
5330 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5331 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5332 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5333
5334 /* Leave VMX Root Mode. */
5335 VMXDisable();
5336
5337 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5338
5339 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5340 CPUMSetHyperEIP(pVCpu, enmOp);
5341 for (int i = (int)cParams - 1; i >= 0; i--)
5342 CPUMPushHyper(pVCpu, paParam[i]);
5343
5344 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5345
5346 /* Call the switcher. */
5347 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5348 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5349
5350 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5351 /* Make sure the VMX instructions don't cause #UD faults. */
5352 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5353
5354 /* Re-enter VMX Root Mode */
5355 int rc2 = VMXEnable(HCPhysCpuPage);
5356 if (RT_FAILURE(rc2))
5357 {
5358 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5359 ASMSetFlags(fOldEFlags);
5360 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5361 return rc2;
5362 }
5363
5364 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5365 AssertRC(rc2);
5366 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5367 Assert(!(ASMGetFlags() & X86_EFL_IF));
5368 ASMSetFlags(fOldEFlags);
5369 return rc;
5370}
5371
5372
5373/**
5374 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5375 * supporting 64-bit guests.
5376 *
5377 * @returns VBox status code.
5378 * @param fResume Whether to VMLAUNCH or VMRESUME.
5379 * @param pCtx Pointer to the guest-CPU context.
5380 * @param pCache Pointer to the VMCS cache.
5381 * @param pVM The cross context VM structure.
5382 * @param pVCpu The cross context virtual CPU structure.
5383 */
5384DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5385{
5386 NOREF(fResume);
5387
5388 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5389 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5390
5391#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5392 pCache->uPos = 1;
5393 pCache->interPD = PGMGetInterPaeCR3(pVM);
5394 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5395#endif
5396
5397#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5398 pCache->TestIn.HCPhysCpuPage = 0;
5399 pCache->TestIn.HCPhysVmcs = 0;
5400 pCache->TestIn.pCache = 0;
5401 pCache->TestOut.HCPhysVmcs = 0;
5402 pCache->TestOut.pCache = 0;
5403 pCache->TestOut.pCtx = 0;
5404 pCache->TestOut.eflags = 0;
5405#else
5406 NOREF(pCache);
5407#endif
5408
5409 uint32_t aParam[10];
5410 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5411 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5412 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5413 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5414 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5415 aParam[5] = 0;
5416 aParam[6] = VM_RC_ADDR(pVM, pVM);
5417 aParam[7] = 0;
5418 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5419 aParam[9] = 0;
5420
5421#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5422 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5423 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5424#endif
5425 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5426
5427#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5428 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5429 Assert(pCtx->dr[4] == 10);
5430 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5431#endif
5432
5433#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5434 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5435 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5436 pVCpu->hm.s.vmx.HCPhysVmcs));
5437 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5438 pCache->TestOut.HCPhysVmcs));
5439 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5440 pCache->TestOut.pCache));
5441 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5442 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5443 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5444 pCache->TestOut.pCtx));
5445 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5446#endif
5447 return rc;
5448}
5449
5450
5451/**
5452 * Initialize the VMCS-Read cache.
5453 *
5454 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5455 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5456 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5457 * (those that have a 32-bit FULL & HIGH part).
5458 *
5459 * @returns VBox status code.
5460 * @param pVM The cross context VM structure.
5461 * @param pVCpu The cross context virtual CPU structure.
5462 */
5463static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5464{
5465#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5466{ \
5467 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5468 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5469 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5470 ++cReadFields; \
5471}
5472
5473 AssertPtr(pVM);
5474 AssertPtr(pVCpu);
5475 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5476 uint32_t cReadFields = 0;
5477
5478 /*
5479 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5480 * and serve to indicate exceptions to the rules.
5481 */
5482
5483 /* Guest-natural selector base fields. */
5484#if 0
5485 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5488#endif
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5496 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5499 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5501#if 0
5502 /* Unused natural width guest-state fields. */
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5505#endif
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5508
5509 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5510#if 0
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5520#endif
5521
5522 /* Natural width guest-state fields. */
5523 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5524#if 0
5525 /* Currently unused field. */
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5527#endif
5528
5529 if (pVM->hm.s.fNestedPaging)
5530 {
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5532 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5533 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5534 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5535 }
5536 else
5537 {
5538 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5539 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5540 }
5541
5542#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5543 return VINF_SUCCESS;
5544}
5545
5546
5547/**
5548 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5549 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5550 * darwin, running 64-bit guests).
5551 *
5552 * @returns VBox status code.
5553 * @param pVCpu The cross context virtual CPU structure.
5554 * @param idxField The VMCS field encoding.
5555 * @param u64Val 16, 32 or 64-bit value.
5556 */
5557VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5558{
5559 int rc;
5560 switch (idxField)
5561 {
5562 /*
5563 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5564 */
5565 /* 64-bit Control fields. */
5566 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5567 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5568 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5569 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5570 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5571 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5572 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5573 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5574 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5575 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5576 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5577 case VMX_VMCS64_CTRL_EPTP_FULL:
5578 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5579 /* 64-bit Guest-state fields. */
5580 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5581 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5582 case VMX_VMCS64_GUEST_PAT_FULL:
5583 case VMX_VMCS64_GUEST_EFER_FULL:
5584 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5585 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5586 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5587 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5588 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5589 /* 64-bit Host-state fields. */
5590 case VMX_VMCS64_HOST_PAT_FULL:
5591 case VMX_VMCS64_HOST_EFER_FULL:
5592 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5593 {
5594 rc = VMXWriteVmcs32(idxField, u64Val);
5595 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5596 break;
5597 }
5598
5599 /*
5600 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5601 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5602 */
5603 /* Natural-width Guest-state fields. */
5604 case VMX_VMCS_GUEST_CR3:
5605 case VMX_VMCS_GUEST_ES_BASE:
5606 case VMX_VMCS_GUEST_CS_BASE:
5607 case VMX_VMCS_GUEST_SS_BASE:
5608 case VMX_VMCS_GUEST_DS_BASE:
5609 case VMX_VMCS_GUEST_FS_BASE:
5610 case VMX_VMCS_GUEST_GS_BASE:
5611 case VMX_VMCS_GUEST_LDTR_BASE:
5612 case VMX_VMCS_GUEST_TR_BASE:
5613 case VMX_VMCS_GUEST_GDTR_BASE:
5614 case VMX_VMCS_GUEST_IDTR_BASE:
5615 case VMX_VMCS_GUEST_RSP:
5616 case VMX_VMCS_GUEST_RIP:
5617 case VMX_VMCS_GUEST_SYSENTER_ESP:
5618 case VMX_VMCS_GUEST_SYSENTER_EIP:
5619 {
5620 if (!(u64Val >> 32))
5621 {
5622 /* If this field is 64-bit, VT-x will zero out the top bits. */
5623 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5624 }
5625 else
5626 {
5627 /* Assert that only the 32->64 switcher case should ever come here. */
5628 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5629 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5630 }
5631 break;
5632 }
5633
5634 default:
5635 {
5636 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5637 rc = VERR_INVALID_PARAMETER;
5638 break;
5639 }
5640 }
5641 AssertRCReturn(rc, rc);
5642 return rc;
5643}
5644
5645
5646/**
5647 * Queue up a VMWRITE by using the VMCS write cache.
5648 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5649 *
5650 * @param pVCpu The cross context virtual CPU structure.
5651 * @param idxField The VMCS field encoding.
5652 * @param u64Val 16, 32 or 64-bit value.
5653 */
5654VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5655{
5656 AssertPtr(pVCpu);
5657 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5658
5659 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5660 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5661
5662 /* Make sure there are no duplicates. */
5663 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5664 {
5665 if (pCache->Write.aField[i] == idxField)
5666 {
5667 pCache->Write.aFieldVal[i] = u64Val;
5668 return VINF_SUCCESS;
5669 }
5670 }
5671
5672 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5673 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5674 pCache->Write.cValidEntries++;
5675 return VINF_SUCCESS;
5676}
5677#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5678
5679
5680/**
5681 * Sets up the usage of TSC-offsetting and updates the VMCS.
5682 *
5683 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5684 * VMX preemption timer.
5685 *
5686 * @returns VBox status code.
5687 * @param pVM The cross context VM structure.
5688 * @param pVCpu The cross context virtual CPU structure.
5689 *
5690 * @remarks No-long-jump zone!!!
5691 */
5692static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5693{
5694 int rc;
5695 bool fOffsettedTsc;
5696 bool fParavirtTsc;
5697 if (pVM->hm.s.vmx.fUsePreemptTimer)
5698 {
5699 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5700 &fOffsettedTsc, &fParavirtTsc);
5701
5702 /* Make sure the returned values have sane upper and lower boundaries. */
5703 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5704 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5705 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5706 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5707
5708 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5709 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5710 }
5711 else
5712 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5713
5714 /** @todo later optimize this to be done elsewhere and not before every
5715 * VM-entry. */
5716 if (fParavirtTsc)
5717 {
5718 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5719 information before every VM-entry, hence disable it for performance sake. */
5720#if 0
5721 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5722 AssertRC(rc);
5723#endif
5724 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5725 }
5726
5727 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5728 {
5729 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5730 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5731
5732 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5733 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5734 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5735 }
5736 else
5737 {
5738 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5739 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5740 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5741 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5742 }
5743}
5744
5745
5746/**
5747 * Determines if an exception is a contributory exception.
5748 *
5749 * Contributory exceptions are ones which can cause double-faults unless the
5750 * original exception was a benign exception. Page-fault is intentionally not
5751 * included here as it's a conditional contributory exception.
5752 *
5753 * @returns true if the exception is contributory, false otherwise.
5754 * @param uVector The exception vector.
5755 */
5756DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5757{
5758 switch (uVector)
5759 {
5760 case X86_XCPT_GP:
5761 case X86_XCPT_SS:
5762 case X86_XCPT_NP:
5763 case X86_XCPT_TS:
5764 case X86_XCPT_DE:
5765 return true;
5766 default:
5767 break;
5768 }
5769 return false;
5770}
5771
5772
5773/**
5774 * Sets an event as a pending event to be injected into the guest.
5775 *
5776 * @param pVCpu The cross context virtual CPU structure.
5777 * @param u32IntInfo The VM-entry interruption-information field.
5778 * @param cbInstr The VM-entry instruction length in bytes (for software
5779 * interrupts, exceptions and privileged software
5780 * exceptions).
5781 * @param u32ErrCode The VM-entry exception error code.
5782 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5783 * page-fault.
5784 *
5785 * @remarks Statistics counter assumes this is a guest event being injected or
5786 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5787 * always incremented.
5788 */
5789DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5790 RTGCUINTPTR GCPtrFaultAddress)
5791{
5792 Assert(!pVCpu->hm.s.Event.fPending);
5793 pVCpu->hm.s.Event.fPending = true;
5794 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5795 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5796 pVCpu->hm.s.Event.cbInstr = cbInstr;
5797 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5798}
5799
5800
5801/**
5802 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5803 *
5804 * @param pVCpu The cross context virtual CPU structure.
5805 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5806 * out-of-sync. Make sure to update the required fields
5807 * before using them.
5808 */
5809DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5810{
5811 NOREF(pMixedCtx);
5812 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5813 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5814 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5815 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5816}
5817
5818
5819/**
5820 * Handle a condition that occurred while delivering an event through the guest
5821 * IDT.
5822 *
5823 * @returns Strict VBox status code (i.e. informational status codes too).
5824 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5825 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5826 * to continue execution of the guest which will delivery the \#DF.
5827 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5828 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5829 *
5830 * @param pVCpu The cross context virtual CPU structure.
5831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5832 * out-of-sync. Make sure to update the required fields
5833 * before using them.
5834 * @param pVmxTransient Pointer to the VMX transient structure.
5835 *
5836 * @remarks No-long-jump zone!!!
5837 */
5838static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5839{
5840 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5841
5842 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5843 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5844
5845 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5846 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5847 {
5848 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5849 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5850
5851 typedef enum
5852 {
5853 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5854 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5855 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5856 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5857 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5858 } VMXREFLECTXCPT;
5859
5860 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5861 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5862 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5863 {
5864 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5865 {
5866 enmReflect = VMXREFLECTXCPT_XCPT;
5867#ifdef VBOX_STRICT
5868 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5869 && uExitVector == X86_XCPT_PF)
5870 {
5871 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5872 }
5873#endif
5874 if ( uExitVector == X86_XCPT_PF
5875 && uIdtVector == X86_XCPT_PF)
5876 {
5877 pVmxTransient->fVectoringDoublePF = true;
5878 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5879 }
5880 else if ( uExitVector == X86_XCPT_AC
5881 && uIdtVector == X86_XCPT_AC)
5882 {
5883 enmReflect = VMXREFLECTXCPT_HANG;
5884 Log4(("IDT: Nested #AC - Bad guest\n"));
5885 }
5886 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5887 && hmR0VmxIsContributoryXcpt(uExitVector)
5888 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5889 || uIdtVector == X86_XCPT_PF))
5890 {
5891 enmReflect = VMXREFLECTXCPT_DF;
5892 }
5893 else if (uIdtVector == X86_XCPT_DF)
5894 enmReflect = VMXREFLECTXCPT_TF;
5895 }
5896 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5897 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5898 {
5899 /*
5900 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5901 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5902 */
5903 enmReflect = VMXREFLECTXCPT_XCPT;
5904
5905 if (uExitVector == X86_XCPT_PF)
5906 {
5907 pVmxTransient->fVectoringPF = true;
5908 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5909 }
5910 }
5911 }
5912 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5913 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5914 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5915 {
5916 /*
5917 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5918 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5919 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5920 */
5921 enmReflect = VMXREFLECTXCPT_XCPT;
5922 }
5923
5924 /*
5925 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5926 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5927 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5928 *
5929 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5930 */
5931 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5932 && enmReflect == VMXREFLECTXCPT_XCPT
5933 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5934 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5935 {
5936 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5937 }
5938
5939 switch (enmReflect)
5940 {
5941 case VMXREFLECTXCPT_XCPT:
5942 {
5943 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5944 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5945 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5946
5947 uint32_t u32ErrCode = 0;
5948 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5949 {
5950 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5951 AssertRCReturn(rc2, rc2);
5952 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5953 }
5954
5955 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5956 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5957 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5958 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5959 rcStrict = VINF_SUCCESS;
5960 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5961 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5962
5963 break;
5964 }
5965
5966 case VMXREFLECTXCPT_DF:
5967 {
5968 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5969 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5970 rcStrict = VINF_HM_DOUBLE_FAULT;
5971 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5972 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5973
5974 break;
5975 }
5976
5977 case VMXREFLECTXCPT_TF:
5978 {
5979 rcStrict = VINF_EM_RESET;
5980 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5981 uExitVector));
5982 break;
5983 }
5984
5985 case VMXREFLECTXCPT_HANG:
5986 {
5987 rcStrict = VERR_EM_GUEST_CPU_HANG;
5988 break;
5989 }
5990
5991 default:
5992 Assert(rcStrict == VINF_SUCCESS);
5993 break;
5994 }
5995 }
5996 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5997 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5998 && uExitVector != X86_XCPT_DF
5999 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6000 {
6001 /*
6002 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6003 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6004 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6005 */
6006 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6007 {
6008 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6009 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6010 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6011 }
6012 }
6013
6014 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6015 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6016 return rcStrict;
6017}
6018
6019
6020/**
6021 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6022 *
6023 * @returns VBox status code.
6024 * @param pVCpu The cross context virtual CPU structure.
6025 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6026 * out-of-sync. Make sure to update the required fields
6027 * before using them.
6028 *
6029 * @remarks No-long-jump zone!!!
6030 */
6031static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6032{
6033 NOREF(pMixedCtx);
6034
6035 /*
6036 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6037 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6038 */
6039 VMMRZCallRing3Disable(pVCpu);
6040 HM_DISABLE_PREEMPT();
6041
6042 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6043 {
6044 uint32_t uVal = 0;
6045 uint32_t uShadow = 0;
6046 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6047 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6048 AssertRCReturn(rc, rc);
6049
6050 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6051 CPUMSetGuestCR0(pVCpu, uVal);
6052 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6053 }
6054
6055 HM_RESTORE_PREEMPT();
6056 VMMRZCallRing3Enable(pVCpu);
6057 return VINF_SUCCESS;
6058}
6059
6060
6061/**
6062 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6063 *
6064 * @returns VBox status code.
6065 * @param pVCpu The cross context virtual CPU structure.
6066 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6067 * out-of-sync. Make sure to update the required fields
6068 * before using them.
6069 *
6070 * @remarks No-long-jump zone!!!
6071 */
6072static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6073{
6074 NOREF(pMixedCtx);
6075
6076 int rc = VINF_SUCCESS;
6077 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6078 {
6079 uint32_t uVal = 0;
6080 uint32_t uShadow = 0;
6081 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6082 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6083 AssertRCReturn(rc, rc);
6084
6085 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6086 CPUMSetGuestCR4(pVCpu, uVal);
6087 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6088 }
6089 return rc;
6090}
6091
6092
6093/**
6094 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6095 *
6096 * @returns VBox status code.
6097 * @param pVCpu The cross context virtual CPU structure.
6098 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6099 * out-of-sync. Make sure to update the required fields
6100 * before using them.
6101 *
6102 * @remarks No-long-jump zone!!!
6103 */
6104static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6105{
6106 int rc = VINF_SUCCESS;
6107 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6108 {
6109 uint64_t u64Val = 0;
6110 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6111 AssertRCReturn(rc, rc);
6112
6113 pMixedCtx->rip = u64Val;
6114 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6115 }
6116 return rc;
6117}
6118
6119
6120/**
6121 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6122 *
6123 * @returns VBox status code.
6124 * @param pVCpu The cross context virtual CPU structure.
6125 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6126 * out-of-sync. Make sure to update the required fields
6127 * before using them.
6128 *
6129 * @remarks No-long-jump zone!!!
6130 */
6131static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6132{
6133 int rc = VINF_SUCCESS;
6134 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6135 {
6136 uint64_t u64Val = 0;
6137 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6138 AssertRCReturn(rc, rc);
6139
6140 pMixedCtx->rsp = u64Val;
6141 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6142 }
6143 return rc;
6144}
6145
6146
6147/**
6148 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6149 *
6150 * @returns VBox status code.
6151 * @param pVCpu The cross context virtual CPU structure.
6152 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6153 * out-of-sync. Make sure to update the required fields
6154 * before using them.
6155 *
6156 * @remarks No-long-jump zone!!!
6157 */
6158static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6159{
6160 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6161 {
6162 uint32_t uVal = 0;
6163 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6164 AssertRCReturn(rc, rc);
6165
6166 pMixedCtx->eflags.u32 = uVal;
6167 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6168 {
6169 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6170 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6171
6172 pMixedCtx->eflags.Bits.u1VM = 0;
6173 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6174 }
6175
6176 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6177 }
6178 return VINF_SUCCESS;
6179}
6180
6181
6182/**
6183 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6184 * guest-CPU context.
6185 */
6186DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6187{
6188 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6189 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6190 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6191 return rc;
6192}
6193
6194
6195/**
6196 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6197 * from the guest-state area in the VMCS.
6198 *
6199 * @param pVCpu The cross context virtual CPU structure.
6200 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6201 * out-of-sync. Make sure to update the required fields
6202 * before using them.
6203 *
6204 * @remarks No-long-jump zone!!!
6205 */
6206static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6207{
6208 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6209 {
6210 uint32_t uIntrState = 0;
6211 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6212 AssertRC(rc);
6213
6214 if (!uIntrState)
6215 {
6216 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6217 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6218
6219 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6220 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6221 }
6222 else
6223 {
6224 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6225 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6226 {
6227 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6228 AssertRC(rc);
6229 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6230 AssertRC(rc);
6231
6232 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6233 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6234 }
6235 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6236 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6237
6238 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6239 {
6240 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6241 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6242 }
6243 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6244 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6245 }
6246
6247 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6248 }
6249}
6250
6251
6252/**
6253 * Saves the guest's activity state.
6254 *
6255 * @returns VBox status code.
6256 * @param pVCpu The cross context virtual CPU structure.
6257 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6258 * out-of-sync. Make sure to update the required fields
6259 * before using them.
6260 *
6261 * @remarks No-long-jump zone!!!
6262 */
6263static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6264{
6265 NOREF(pMixedCtx);
6266 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6267 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6268 return VINF_SUCCESS;
6269}
6270
6271
6272/**
6273 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6274 * the current VMCS into the guest-CPU context.
6275 *
6276 * @returns VBox status code.
6277 * @param pVCpu The cross context virtual CPU structure.
6278 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6279 * out-of-sync. Make sure to update the required fields
6280 * before using them.
6281 *
6282 * @remarks No-long-jump zone!!!
6283 */
6284static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6285{
6286 int rc = VINF_SUCCESS;
6287 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6288 {
6289 uint32_t u32Val = 0;
6290 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6291 pMixedCtx->SysEnter.cs = u32Val;
6292 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6293 }
6294
6295 uint64_t u64Val = 0;
6296 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6297 {
6298 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6299 pMixedCtx->SysEnter.eip = u64Val;
6300 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6301 }
6302 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6303 {
6304 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6305 pMixedCtx->SysEnter.esp = u64Val;
6306 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6307 }
6308 return rc;
6309}
6310
6311
6312/**
6313 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6314 * the CPU back into the guest-CPU context.
6315 *
6316 * @returns VBox status code.
6317 * @param pVCpu The cross context virtual CPU structure.
6318 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6319 * out-of-sync. Make sure to update the required fields
6320 * before using them.
6321 *
6322 * @remarks No-long-jump zone!!!
6323 */
6324static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6325{
6326 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6327 VMMRZCallRing3Disable(pVCpu);
6328 HM_DISABLE_PREEMPT();
6329
6330 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6331 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6332 {
6333 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6334 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6335 }
6336
6337 HM_RESTORE_PREEMPT();
6338 VMMRZCallRing3Enable(pVCpu);
6339
6340 return VINF_SUCCESS;
6341}
6342
6343
6344/**
6345 * Saves the auto load/store'd guest MSRs from the current VMCS into
6346 * the guest-CPU context.
6347 *
6348 * @returns VBox status code.
6349 * @param pVCpu The cross context virtual CPU structure.
6350 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6351 * out-of-sync. Make sure to update the required fields
6352 * before using them.
6353 *
6354 * @remarks No-long-jump zone!!!
6355 */
6356static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6357{
6358 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6359 return VINF_SUCCESS;
6360
6361 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6362 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6363 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6364 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6365 {
6366 switch (pMsr->u32Msr)
6367 {
6368 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6369 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6370 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6371 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6372 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6373 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6374 break;
6375
6376 default:
6377 {
6378 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6379 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6380 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6381 }
6382 }
6383 }
6384
6385 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6386 return VINF_SUCCESS;
6387}
6388
6389
6390/**
6391 * Saves the guest control registers from the current VMCS into the guest-CPU
6392 * context.
6393 *
6394 * @returns VBox status code.
6395 * @param pVCpu The cross context virtual CPU structure.
6396 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6397 * out-of-sync. Make sure to update the required fields
6398 * before using them.
6399 *
6400 * @remarks No-long-jump zone!!!
6401 */
6402static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6403{
6404 /* Guest CR0. Guest FPU. */
6405 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6406 AssertRCReturn(rc, rc);
6407
6408 /* Guest CR4. */
6409 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6410 AssertRCReturn(rc, rc);
6411
6412 /* Guest CR2 - updated always during the world-switch or in #PF. */
6413 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6414 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6415 {
6416 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6417 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6418
6419 PVM pVM = pVCpu->CTX_SUFF(pVM);
6420 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6421 || ( pVM->hm.s.fNestedPaging
6422 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6423 {
6424 uint64_t u64Val = 0;
6425 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6426 if (pMixedCtx->cr3 != u64Val)
6427 {
6428 CPUMSetGuestCR3(pVCpu, u64Val);
6429 if (VMMRZCallRing3IsEnabled(pVCpu))
6430 {
6431 PGMUpdateCR3(pVCpu, u64Val);
6432 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6433 }
6434 else
6435 {
6436 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6437 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6438 }
6439 }
6440
6441 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6442 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6443 {
6444 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6445 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6446 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6447 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6448 AssertRCReturn(rc, rc);
6449
6450 if (VMMRZCallRing3IsEnabled(pVCpu))
6451 {
6452 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6453 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6454 }
6455 else
6456 {
6457 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6458 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6459 }
6460 }
6461 }
6462
6463 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6464 }
6465
6466 /*
6467 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6468 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6469 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6470 *
6471 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6472 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6473 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6474 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6475 *
6476 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6477 */
6478 if (VMMRZCallRing3IsEnabled(pVCpu))
6479 {
6480 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6481 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6482
6483 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6484 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6485
6486 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6487 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6488 }
6489
6490 return rc;
6491}
6492
6493
6494/**
6495 * Reads a guest segment register from the current VMCS into the guest-CPU
6496 * context.
6497 *
6498 * @returns VBox status code.
6499 * @param pVCpu The cross context virtual CPU structure.
6500 * @param idxSel Index of the selector in the VMCS.
6501 * @param idxLimit Index of the segment limit in the VMCS.
6502 * @param idxBase Index of the segment base in the VMCS.
6503 * @param idxAccess Index of the access rights of the segment in the VMCS.
6504 * @param pSelReg Pointer to the segment selector.
6505 *
6506 * @remarks No-long-jump zone!!!
6507 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6508 * macro as that takes care of whether to read from the VMCS cache or
6509 * not.
6510 */
6511DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6512 PCPUMSELREG pSelReg)
6513{
6514 NOREF(pVCpu);
6515
6516 uint32_t u32Val = 0;
6517 int rc = VMXReadVmcs32(idxSel, &u32Val);
6518 AssertRCReturn(rc, rc);
6519 pSelReg->Sel = (uint16_t)u32Val;
6520 pSelReg->ValidSel = (uint16_t)u32Val;
6521 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6522
6523 rc = VMXReadVmcs32(idxLimit, &u32Val);
6524 AssertRCReturn(rc, rc);
6525 pSelReg->u32Limit = u32Val;
6526
6527 uint64_t u64Val = 0;
6528 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6529 AssertRCReturn(rc, rc);
6530 pSelReg->u64Base = u64Val;
6531
6532 rc = VMXReadVmcs32(idxAccess, &u32Val);
6533 AssertRCReturn(rc, rc);
6534 pSelReg->Attr.u = u32Val;
6535
6536 /*
6537 * If VT-x marks the segment as unusable, most other bits remain undefined:
6538 * - For CS the L, D and G bits have meaning.
6539 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6540 * - For the remaining data segments no bits are defined.
6541 *
6542 * The present bit and the unusable bit has been observed to be set at the
6543 * same time (the selector was supposed to be invalid as we started executing
6544 * a V8086 interrupt in ring-0).
6545 *
6546 * What should be important for the rest of the VBox code, is that the P bit is
6547 * cleared. Some of the other VBox code recognizes the unusable bit, but
6548 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6549 * safe side here, we'll strip off P and other bits we don't care about. If
6550 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6551 *
6552 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6553 */
6554 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6555 {
6556 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6557
6558 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6559 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6560 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6561
6562 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6563#ifdef DEBUG_bird
6564 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6565 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6566 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6567#endif
6568 }
6569 return VINF_SUCCESS;
6570}
6571
6572
6573#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6574# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6575 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6576 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6577#else
6578# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6579 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6580 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6581#endif
6582
6583
6584/**
6585 * Saves the guest segment registers from the current VMCS into the guest-CPU
6586 * context.
6587 *
6588 * @returns VBox status code.
6589 * @param pVCpu The cross context virtual CPU structure.
6590 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6591 * out-of-sync. Make sure to update the required fields
6592 * before using them.
6593 *
6594 * @remarks No-long-jump zone!!!
6595 */
6596static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6597{
6598 /* Guest segment registers. */
6599 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6600 {
6601 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6602 AssertRCReturn(rc, rc);
6603
6604 rc = VMXLOCAL_READ_SEG(CS, cs);
6605 rc |= VMXLOCAL_READ_SEG(SS, ss);
6606 rc |= VMXLOCAL_READ_SEG(DS, ds);
6607 rc |= VMXLOCAL_READ_SEG(ES, es);
6608 rc |= VMXLOCAL_READ_SEG(FS, fs);
6609 rc |= VMXLOCAL_READ_SEG(GS, gs);
6610 AssertRCReturn(rc, rc);
6611
6612 /* Restore segment attributes for real-on-v86 mode hack. */
6613 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6614 {
6615 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6616 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6617 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6618 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6619 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6620 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6621 }
6622 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6623 }
6624
6625 return VINF_SUCCESS;
6626}
6627
6628
6629/**
6630 * Saves the guest descriptor table registers and task register from the current
6631 * VMCS into the guest-CPU context.
6632 *
6633 * @returns VBox status code.
6634 * @param pVCpu The cross context virtual CPU structure.
6635 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6636 * out-of-sync. Make sure to update the required fields
6637 * before using them.
6638 *
6639 * @remarks No-long-jump zone!!!
6640 */
6641static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6642{
6643 int rc = VINF_SUCCESS;
6644
6645 /* Guest LDTR. */
6646 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6647 {
6648 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6649 AssertRCReturn(rc, rc);
6650 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6651 }
6652
6653 /* Guest GDTR. */
6654 uint64_t u64Val = 0;
6655 uint32_t u32Val = 0;
6656 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6657 {
6658 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6659 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6660 pMixedCtx->gdtr.pGdt = u64Val;
6661 pMixedCtx->gdtr.cbGdt = u32Val;
6662 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6663 }
6664
6665 /* Guest IDTR. */
6666 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6667 {
6668 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6669 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6670 pMixedCtx->idtr.pIdt = u64Val;
6671 pMixedCtx->idtr.cbIdt = u32Val;
6672 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6673 }
6674
6675 /* Guest TR. */
6676 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6677 {
6678 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6679 AssertRCReturn(rc, rc);
6680
6681 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6682 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6683 {
6684 rc = VMXLOCAL_READ_SEG(TR, tr);
6685 AssertRCReturn(rc, rc);
6686 }
6687 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6688 }
6689 return rc;
6690}
6691
6692#undef VMXLOCAL_READ_SEG
6693
6694
6695/**
6696 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6697 * context.
6698 *
6699 * @returns VBox status code.
6700 * @param pVCpu The cross context virtual CPU structure.
6701 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6702 * out-of-sync. Make sure to update the required fields
6703 * before using them.
6704 *
6705 * @remarks No-long-jump zone!!!
6706 */
6707static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6708{
6709 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6710 {
6711 if (!pVCpu->hm.s.fUsingHyperDR7)
6712 {
6713 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6714 uint32_t u32Val;
6715 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6716 pMixedCtx->dr[7] = u32Val;
6717 }
6718
6719 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6720 }
6721 return VINF_SUCCESS;
6722}
6723
6724
6725/**
6726 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6727 *
6728 * @returns VBox status code.
6729 * @param pVCpu The cross context virtual CPU structure.
6730 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6731 * out-of-sync. Make sure to update the required fields
6732 * before using them.
6733 *
6734 * @remarks No-long-jump zone!!!
6735 */
6736static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6737{
6738 NOREF(pMixedCtx);
6739
6740 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6741 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6742 return VINF_SUCCESS;
6743}
6744
6745
6746/**
6747 * Saves the entire guest state from the currently active VMCS into the
6748 * guest-CPU context.
6749 *
6750 * This essentially VMREADs all guest-data.
6751 *
6752 * @returns VBox status code.
6753 * @param pVCpu The cross context virtual CPU structure.
6754 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6755 * out-of-sync. Make sure to update the required fields
6756 * before using them.
6757 */
6758static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6759{
6760 Assert(pVCpu);
6761 Assert(pMixedCtx);
6762
6763 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6764 return VINF_SUCCESS;
6765
6766 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6767 again on the ring-3 callback path, there is no real need to. */
6768 if (VMMRZCallRing3IsEnabled(pVCpu))
6769 VMMR0LogFlushDisable(pVCpu);
6770 else
6771 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6772 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6773
6774 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6775 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6776
6777 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6778 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6779
6780 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6781 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6782
6783 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6784 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6785
6786 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6787 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6788
6789 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6790 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6791
6792 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6793 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6794
6795 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6796 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6797
6798 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6799 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6800
6801 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6802 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6803
6804 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6805 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6806 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6807
6808 if (VMMRZCallRing3IsEnabled(pVCpu))
6809 VMMR0LogFlushEnable(pVCpu);
6810
6811 return VINF_SUCCESS;
6812}
6813
6814
6815/**
6816 * Saves basic guest registers needed for IEM instruction execution.
6817 *
6818 * @returns VBox status code (OR-able).
6819 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6820 * @param pMixedCtx Pointer to the CPU context of the guest.
6821 * @param fMemory Whether the instruction being executed operates on
6822 * memory or not. Only CR0 is synced up if clear.
6823 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6824 */
6825static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6826{
6827 /*
6828 * We assume all general purpose registers other than RSP are available.
6829 *
6830 * RIP is a must, as it will be incremented or otherwise changed.
6831 *
6832 * RFLAGS are always required to figure the CPL.
6833 *
6834 * RSP isn't always required, however it's a GPR, so frequently required.
6835 *
6836 * SS and CS are the only segment register needed if IEM doesn't do memory
6837 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6838 *
6839 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6840 * be required for memory accesses.
6841 *
6842 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6843 */
6844 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6845 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6846 if (fNeedRsp)
6847 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6848 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6849 if (!fMemory)
6850 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6851 else
6852 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6853 AssertRCReturn(rc, rc);
6854 return rc;
6855}
6856
6857
6858/**
6859 * Ensures that we've got a complete basic guest-context.
6860 *
6861 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6862 * is for the interpreter.
6863 *
6864 * @returns VBox status code.
6865 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6866 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6867 * needing to be synced in.
6868 * @thread EMT(pVCpu)
6869 */
6870VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6871{
6872 /* Note! Since this is only applicable to VT-x, the implementation is placed
6873 in the VT-x part of the sources instead of the generic stuff. */
6874 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6875 {
6876 /* For now, imply that the caller might change everything too. */
6877 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6878 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6879 }
6880 return VINF_SUCCESS;
6881}
6882
6883
6884/**
6885 * Check per-VM and per-VCPU force flag actions that require us to go back to
6886 * ring-3 for one reason or another.
6887 *
6888 * @returns Strict VBox status code (i.e. informational status codes too)
6889 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6890 * ring-3.
6891 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6892 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6893 * interrupts)
6894 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6895 * all EMTs to be in ring-3.
6896 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6897 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6898 * to the EM loop.
6899 *
6900 * @param pVM The cross context VM structure.
6901 * @param pVCpu The cross context virtual CPU structure.
6902 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6903 * out-of-sync. Make sure to update the required fields
6904 * before using them.
6905 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6906 */
6907static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6908{
6909 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6910
6911 /*
6912 * Anything pending? Should be more likely than not if we're doing a good job.
6913 */
6914 if ( !fStepping
6915 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6916 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6917 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6918 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6919 return VINF_SUCCESS;
6920
6921 /* We need the control registers now, make sure the guest-CPU context is updated. */
6922 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6923 AssertRCReturn(rc3, rc3);
6924
6925 /* Pending HM CR3 sync. */
6926 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6927 {
6928 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6929 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6930 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6931 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6932 }
6933
6934 /* Pending HM PAE PDPEs. */
6935 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6936 {
6937 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6938 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6939 }
6940
6941 /* Pending PGM C3 sync. */
6942 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6943 {
6944 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6945 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6946 if (rcStrict2 != VINF_SUCCESS)
6947 {
6948 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6949 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6950 return rcStrict2;
6951 }
6952 }
6953
6954 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6955 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6956 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6957 {
6958 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6959 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6960 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6961 return rc2;
6962 }
6963
6964 /* Pending VM request packets, such as hardware interrupts. */
6965 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6966 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6967 {
6968 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6969 return VINF_EM_PENDING_REQUEST;
6970 }
6971
6972 /* Pending PGM pool flushes. */
6973 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6974 {
6975 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6976 return VINF_PGM_POOL_FLUSH_PENDING;
6977 }
6978
6979 /* Pending DMA requests. */
6980 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6981 {
6982 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6983 return VINF_EM_RAW_TO_R3;
6984 }
6985
6986 return VINF_SUCCESS;
6987}
6988
6989
6990/**
6991 * Converts any TRPM trap into a pending HM event. This is typically used when
6992 * entering from ring-3 (not longjmp returns).
6993 *
6994 * @param pVCpu The cross context virtual CPU structure.
6995 */
6996static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6997{
6998 Assert(TRPMHasTrap(pVCpu));
6999 Assert(!pVCpu->hm.s.Event.fPending);
7000
7001 uint8_t uVector;
7002 TRPMEVENT enmTrpmEvent;
7003 RTGCUINT uErrCode;
7004 RTGCUINTPTR GCPtrFaultAddress;
7005 uint8_t cbInstr;
7006
7007 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7008 AssertRC(rc);
7009
7010 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7011 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7012 if (enmTrpmEvent == TRPM_TRAP)
7013 {
7014 switch (uVector)
7015 {
7016 case X86_XCPT_NMI:
7017 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7018 break;
7019
7020 case X86_XCPT_BP:
7021 case X86_XCPT_OF:
7022 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7023 break;
7024
7025 case X86_XCPT_PF:
7026 case X86_XCPT_DF:
7027 case X86_XCPT_TS:
7028 case X86_XCPT_NP:
7029 case X86_XCPT_SS:
7030 case X86_XCPT_GP:
7031 case X86_XCPT_AC:
7032 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7033 /* no break! */
7034 default:
7035 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7036 break;
7037 }
7038 }
7039 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7040 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7041 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7042 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7043 else
7044 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7045
7046 rc = TRPMResetTrap(pVCpu);
7047 AssertRC(rc);
7048 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7049 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7050
7051 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7052}
7053
7054
7055/**
7056 * Converts the pending HM event into a TRPM trap.
7057 *
7058 * @param pVCpu The cross context virtual CPU structure.
7059 */
7060static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7061{
7062 Assert(pVCpu->hm.s.Event.fPending);
7063
7064 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7065 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7066 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7067 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7068
7069 /* If a trap was already pending, we did something wrong! */
7070 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7071
7072 TRPMEVENT enmTrapType;
7073 switch (uVectorType)
7074 {
7075 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7076 enmTrapType = TRPM_HARDWARE_INT;
7077 break;
7078
7079 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7080 enmTrapType = TRPM_SOFTWARE_INT;
7081 break;
7082
7083 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7084 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7085 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7086 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7087 enmTrapType = TRPM_TRAP;
7088 break;
7089
7090 default:
7091 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7092 enmTrapType = TRPM_32BIT_HACK;
7093 break;
7094 }
7095
7096 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7097
7098 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7099 AssertRC(rc);
7100
7101 if (fErrorCodeValid)
7102 TRPMSetErrorCode(pVCpu, uErrorCode);
7103
7104 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7105 && uVector == X86_XCPT_PF)
7106 {
7107 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7108 }
7109 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7110 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7111 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7112 {
7113 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7114 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7115 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7116 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7117 }
7118
7119 /* Clear any pending events from the VMCS. */
7120 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7121 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7122
7123 /* We're now done converting the pending event. */
7124 pVCpu->hm.s.Event.fPending = false;
7125}
7126
7127
7128/**
7129 * Does the necessary state syncing before returning to ring-3 for any reason
7130 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7131 *
7132 * @returns VBox status code.
7133 * @param pVCpu The cross context virtual CPU structure.
7134 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7135 * be out-of-sync. Make sure to update the required
7136 * fields before using them.
7137 * @param fSaveGuestState Whether to save the guest state or not.
7138 *
7139 * @remarks No-long-jmp zone!!!
7140 */
7141static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7142{
7143 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7144 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7145
7146 RTCPUID idCpu = RTMpCpuId();
7147 Log4Func(("HostCpuId=%u\n", idCpu));
7148
7149 /*
7150 * !!! IMPORTANT !!!
7151 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7152 */
7153
7154 /* Save the guest state if necessary. */
7155 if ( fSaveGuestState
7156 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7157 {
7158 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7159 AssertRCReturn(rc, rc);
7160 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7161 }
7162
7163 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7164 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7165 {
7166 if (fSaveGuestState)
7167 {
7168 /* We shouldn't reload CR0 without saving it first. */
7169 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7170 AssertRCReturn(rc, rc);
7171 }
7172 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7173 }
7174
7175 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7176#ifdef VBOX_STRICT
7177 if (CPUMIsHyperDebugStateActive(pVCpu))
7178 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7179#endif
7180 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7181 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7182 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7183 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7184
7185#if HC_ARCH_BITS == 64
7186 /* Restore host-state bits that VT-x only restores partially. */
7187 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7188 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7189 {
7190 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7191 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7192 }
7193 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7194#endif
7195
7196 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7197 if (pVCpu->hm.s.vmx.fLazyMsrs)
7198 {
7199 /* We shouldn't reload the guest MSRs without saving it first. */
7200 if (!fSaveGuestState)
7201 {
7202 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7203 AssertRCReturn(rc, rc);
7204 }
7205 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7206 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7207 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7208 }
7209
7210 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7211 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7212
7213 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7214 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7215 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7216 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7217 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7218 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7219 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7220 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7221
7222 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7223
7224 /** @todo This partially defeats the purpose of having preemption hooks.
7225 * The problem is, deregistering the hooks should be moved to a place that
7226 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7227 * context.
7228 */
7229 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7230 {
7231 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7232 AssertRCReturn(rc, rc);
7233
7234 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7235 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7236 }
7237 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7238 NOREF(idCpu);
7239
7240 return VINF_SUCCESS;
7241}
7242
7243
7244/**
7245 * Leaves the VT-x session.
7246 *
7247 * @returns VBox status code.
7248 * @param pVCpu The cross context virtual CPU structure.
7249 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7250 * out-of-sync. Make sure to update the required fields
7251 * before using them.
7252 *
7253 * @remarks No-long-jmp zone!!!
7254 */
7255DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7256{
7257 HM_DISABLE_PREEMPT();
7258 HMVMX_ASSERT_CPU_SAFE();
7259 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7260 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7261
7262 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7263 and done this from the VMXR0ThreadCtxCallback(). */
7264 if (!pVCpu->hm.s.fLeaveDone)
7265 {
7266 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7267 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7268 pVCpu->hm.s.fLeaveDone = true;
7269 }
7270 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7271
7272 /*
7273 * !!! IMPORTANT !!!
7274 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7275 */
7276
7277 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7278 /** @todo Deregistering here means we need to VMCLEAR always
7279 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7280 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7281 VMMR0ThreadCtxHookDisable(pVCpu);
7282
7283 /* Leave HM context. This takes care of local init (term). */
7284 int rc = HMR0LeaveCpu(pVCpu);
7285
7286 HM_RESTORE_PREEMPT();
7287 return rc;
7288}
7289
7290
7291/**
7292 * Does the necessary state syncing before doing a longjmp to ring-3.
7293 *
7294 * @returns VBox status code.
7295 * @param pVCpu The cross context virtual CPU structure.
7296 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7297 * out-of-sync. Make sure to update the required fields
7298 * before using them.
7299 *
7300 * @remarks No-long-jmp zone!!!
7301 */
7302DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7303{
7304 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7305}
7306
7307
7308/**
7309 * Take necessary actions before going back to ring-3.
7310 *
7311 * An action requires us to go back to ring-3. This function does the necessary
7312 * steps before we can safely return to ring-3. This is not the same as longjmps
7313 * to ring-3, this is voluntary and prepares the guest so it may continue
7314 * executing outside HM (recompiler/IEM).
7315 *
7316 * @returns VBox status code.
7317 * @param pVM The cross context VM structure.
7318 * @param pVCpu The cross context virtual CPU structure.
7319 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7320 * out-of-sync. Make sure to update the required fields
7321 * before using them.
7322 * @param rcExit The reason for exiting to ring-3. Can be
7323 * VINF_VMM_UNKNOWN_RING3_CALL.
7324 */
7325static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7326{
7327 Assert(pVM);
7328 Assert(pVCpu);
7329 Assert(pMixedCtx);
7330 HMVMX_ASSERT_PREEMPT_SAFE();
7331
7332 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7333 {
7334 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7335 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7336 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7337 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7338 }
7339
7340 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7341 VMMRZCallRing3Disable(pVCpu);
7342 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7343
7344 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7345 if (pVCpu->hm.s.Event.fPending)
7346 {
7347 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7348 Assert(!pVCpu->hm.s.Event.fPending);
7349 }
7350
7351 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7352 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7353
7354 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7355 and if we're injecting an event we should have a TRPM trap pending. */
7356 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7357#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7358 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7359#endif
7360
7361 /* Save guest state and restore host state bits. */
7362 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7363 AssertRCReturn(rc, rc);
7364 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7365 /* Thread-context hooks are unregistered at this point!!! */
7366
7367 /* Sync recompiler state. */
7368 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7369 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7370 | CPUM_CHANGED_LDTR
7371 | CPUM_CHANGED_GDTR
7372 | CPUM_CHANGED_IDTR
7373 | CPUM_CHANGED_TR
7374 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7375 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7376 if ( pVM->hm.s.fNestedPaging
7377 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7378 {
7379 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7380 }
7381
7382 Assert(!pVCpu->hm.s.fClearTrapFlag);
7383
7384 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7385 if (rcExit != VINF_EM_RAW_INTERRUPT)
7386 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7387
7388 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7389
7390 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7391 VMMRZCallRing3RemoveNotification(pVCpu);
7392 VMMRZCallRing3Enable(pVCpu);
7393
7394 return rc;
7395}
7396
7397
7398/**
7399 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7400 * longjump to ring-3 and possibly get preempted.
7401 *
7402 * @returns VBox status code.
7403 * @param pVCpu The cross context virtual CPU structure.
7404 * @param enmOperation The operation causing the ring-3 longjump.
7405 * @param pvUser Opaque pointer to the guest-CPU context. The data
7406 * may be out-of-sync. Make sure to update the required
7407 * fields before using them.
7408 */
7409static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7410{
7411 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7412 {
7413 /*
7414 * !!! IMPORTANT !!!
7415 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7416 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7417 */
7418 VMMRZCallRing3RemoveNotification(pVCpu);
7419 VMMRZCallRing3Disable(pVCpu);
7420 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7421 RTThreadPreemptDisable(&PreemptState);
7422
7423 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7424 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7425
7426#if HC_ARCH_BITS == 64
7427 /* Restore host-state bits that VT-x only restores partially. */
7428 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7429 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7430 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7431 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7432#endif
7433 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7434 if (pVCpu->hm.s.vmx.fLazyMsrs)
7435 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7436
7437 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7438 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7439 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7440 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7441 {
7442 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7443 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7444 }
7445
7446 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7447 VMMR0ThreadCtxHookDisable(pVCpu);
7448 HMR0LeaveCpu(pVCpu);
7449 RTThreadPreemptRestore(&PreemptState);
7450 return VINF_SUCCESS;
7451 }
7452
7453 Assert(pVCpu);
7454 Assert(pvUser);
7455 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7456 HMVMX_ASSERT_PREEMPT_SAFE();
7457
7458 VMMRZCallRing3Disable(pVCpu);
7459 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7460
7461 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7462 enmOperation));
7463
7464 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7465 AssertRCReturn(rc, rc);
7466
7467 VMMRZCallRing3Enable(pVCpu);
7468 return VINF_SUCCESS;
7469}
7470
7471
7472/**
7473 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7474 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7475 *
7476 * @param pVCpu The cross context virtual CPU structure.
7477 */
7478DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7479{
7480 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7481 {
7482 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7483 {
7484 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7485 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7486 AssertRC(rc);
7487 Log4(("Setup interrupt-window exiting\n"));
7488 }
7489 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7490}
7491
7492
7493/**
7494 * Clears the interrupt-window exiting control in the VMCS.
7495 *
7496 * @param pVCpu The cross context virtual CPU structure.
7497 */
7498DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7499{
7500 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7501 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7502 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7503 AssertRC(rc);
7504 Log4(("Cleared interrupt-window exiting\n"));
7505}
7506
7507
7508/**
7509 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7510 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7511 *
7512 * @param pVCpu The cross context virtual CPU structure.
7513 */
7514DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7515{
7516 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7517 {
7518 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7519 {
7520 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7521 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7522 AssertRC(rc);
7523 Log4(("Setup NMI-window exiting\n"));
7524 }
7525 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7526}
7527
7528
7529/**
7530 * Clears the NMI-window exiting control in the VMCS.
7531 *
7532 * @param pVCpu The cross context virtual CPU structure.
7533 */
7534DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7535{
7536 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7537 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7538 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7539 AssertRC(rc);
7540 Log4(("Cleared NMI-window exiting\n"));
7541}
7542
7543
7544/**
7545 * Evaluates the event to be delivered to the guest and sets it as the pending
7546 * event.
7547 *
7548 * @returns The VT-x guest-interruptibility state.
7549 * @param pVCpu The cross context virtual CPU structure.
7550 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7551 * out-of-sync. Make sure to update the required fields
7552 * before using them.
7553 */
7554static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7555{
7556 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7557 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7558 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7559 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7560 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7561
7562 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7563 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7564 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7565 Assert(!TRPMHasTrap(pVCpu));
7566
7567 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7568 APICUpdatePendingInterrupts(pVCpu);
7569
7570 /*
7571 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7572 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7573 */
7574 /** @todo SMI. SMIs take priority over NMIs. */
7575 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7576 {
7577 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7578 if ( !pVCpu->hm.s.Event.fPending
7579 && !fBlockNmi
7580 && !fBlockSti
7581 && !fBlockMovSS)
7582 {
7583 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7584 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7585 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7586
7587 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7588 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7589 }
7590 else
7591 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7592 }
7593 /*
7594 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7595 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7596 */
7597 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7598 && !pVCpu->hm.s.fSingleInstruction)
7599 {
7600 Assert(!DBGFIsStepping(pVCpu));
7601 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7602 AssertRC(rc);
7603 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7604 if ( !pVCpu->hm.s.Event.fPending
7605 && !fBlockInt
7606 && !fBlockSti
7607 && !fBlockMovSS)
7608 {
7609 uint8_t u8Interrupt;
7610 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7611 if (RT_SUCCESS(rc))
7612 {
7613 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7614 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7615 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7616
7617 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7618 }
7619 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7620 {
7621 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7622 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7623 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7624
7625 /*
7626 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7627 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7628 * need to re-set this force-flag here.
7629 */
7630 }
7631 else
7632 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7633 }
7634 else
7635 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7636 }
7637
7638 return uIntrState;
7639}
7640
7641
7642/**
7643 * Sets a pending-debug exception to be delivered to the guest if the guest is
7644 * single-stepping in the VMCS.
7645 *
7646 * @param pVCpu The cross context virtual CPU structure.
7647 */
7648DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7649{
7650 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7651 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7652 AssertRC(rc);
7653}
7654
7655
7656/**
7657 * Injects any pending events into the guest if the guest is in a state to
7658 * receive them.
7659 *
7660 * @returns Strict VBox status code (i.e. informational status codes too).
7661 * @param pVCpu The cross context virtual CPU structure.
7662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7663 * out-of-sync. Make sure to update the required fields
7664 * before using them.
7665 * @param uIntrState The VT-x guest-interruptibility state.
7666 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7667 * return VINF_EM_DBG_STEPPED if the event was
7668 * dispatched directly.
7669 */
7670static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7671{
7672 HMVMX_ASSERT_PREEMPT_SAFE();
7673 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7674
7675 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7676 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7677
7678 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7679 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7680 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7681 Assert(!TRPMHasTrap(pVCpu));
7682
7683 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7684 if (pVCpu->hm.s.Event.fPending)
7685 {
7686 /*
7687 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7688 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7689 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7690 *
7691 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7692 */
7693 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7694#ifdef VBOX_STRICT
7695 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7696 {
7697 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7698 Assert(!fBlockInt);
7699 Assert(!fBlockSti);
7700 Assert(!fBlockMovSS);
7701 }
7702 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7703 {
7704 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7705 Assert(!fBlockSti);
7706 Assert(!fBlockMovSS);
7707 Assert(!fBlockNmi);
7708 }
7709#endif
7710 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7711 (uint8_t)uIntType));
7712 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7713 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7714 fStepping, &uIntrState);
7715 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7716
7717 /* Update the interruptibility-state as it could have been changed by
7718 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7719 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7720 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7721
7722 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7723 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7724 else
7725 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7726 }
7727
7728 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7729 if ( fBlockSti
7730 || fBlockMovSS)
7731 {
7732 if (!pVCpu->hm.s.fSingleInstruction)
7733 {
7734 /*
7735 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7736 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7737 * See Intel spec. 27.3.4 "Saving Non-Register State".
7738 */
7739 Assert(!DBGFIsStepping(pVCpu));
7740 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7741 AssertRCReturn(rc2, rc2);
7742 if (pMixedCtx->eflags.Bits.u1TF)
7743 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7744 }
7745 else if (pMixedCtx->eflags.Bits.u1TF)
7746 {
7747 /*
7748 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7749 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7750 */
7751 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7752 uIntrState = 0;
7753 }
7754 }
7755
7756 /*
7757 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7758 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7759 */
7760 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7761 AssertRC(rc2);
7762
7763 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7764 NOREF(fBlockMovSS); NOREF(fBlockSti);
7765 return rcStrict;
7766}
7767
7768
7769/**
7770 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7771 *
7772 * @param pVCpu The cross context virtual CPU structure.
7773 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7774 * out-of-sync. Make sure to update the required fields
7775 * before using them.
7776 */
7777DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7778{
7779 NOREF(pMixedCtx);
7780 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7781 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7782}
7783
7784
7785/**
7786 * Injects a double-fault (\#DF) exception into the VM.
7787 *
7788 * @returns Strict VBox status code (i.e. informational status codes too).
7789 * @param pVCpu The cross context virtual CPU structure.
7790 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7791 * out-of-sync. Make sure to update the required fields
7792 * before using them.
7793 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7794 * and should return VINF_EM_DBG_STEPPED if the event
7795 * is injected directly (register modified by us, not
7796 * by hardware on VM-entry).
7797 * @param puIntrState Pointer to the current guest interruptibility-state.
7798 * This interruptibility-state will be updated if
7799 * necessary. This cannot not be NULL.
7800 */
7801DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7802{
7803 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7804 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7805 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7806 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7807 fStepping, puIntrState);
7808}
7809
7810
7811/**
7812 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7813 *
7814 * @param pVCpu The cross context virtual CPU structure.
7815 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7816 * out-of-sync. Make sure to update the required fields
7817 * before using them.
7818 */
7819DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7820{
7821 NOREF(pMixedCtx);
7822 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7823 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7824 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7825}
7826
7827
7828/**
7829 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7830 *
7831 * @param pVCpu The cross context virtual CPU structure.
7832 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7833 * out-of-sync. Make sure to update the required fields
7834 * before using them.
7835 * @param cbInstr The value of RIP that is to be pushed on the guest
7836 * stack.
7837 */
7838DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7839{
7840 NOREF(pMixedCtx);
7841 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7842 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7843 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7844}
7845
7846
7847/**
7848 * Injects a general-protection (\#GP) fault into the VM.
7849 *
7850 * @returns Strict VBox status code (i.e. informational status codes too).
7851 * @param pVCpu The cross context virtual CPU structure.
7852 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7853 * out-of-sync. Make sure to update the required fields
7854 * before using them.
7855 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7856 * mode, i.e. in real-mode it's not valid).
7857 * @param u32ErrorCode The error code associated with the \#GP.
7858 * @param fStepping Whether we're running in
7859 * hmR0VmxRunGuestCodeStep() and should return
7860 * VINF_EM_DBG_STEPPED if the event is injected
7861 * directly (register modified by us, not by
7862 * hardware on VM-entry).
7863 * @param puIntrState Pointer to the current guest interruptibility-state.
7864 * This interruptibility-state will be updated if
7865 * necessary. This cannot not be NULL.
7866 */
7867DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7868 bool fStepping, uint32_t *puIntrState)
7869{
7870 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7871 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7872 if (fErrorCodeValid)
7873 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7874 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7875 fStepping, puIntrState);
7876}
7877
7878
7879#if 0 /* unused */
7880/**
7881 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7882 * VM.
7883 *
7884 * @param pVCpu The cross context virtual CPU structure.
7885 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7886 * out-of-sync. Make sure to update the required fields
7887 * before using them.
7888 * @param u32ErrorCode The error code associated with the \#GP.
7889 */
7890DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7891{
7892 NOREF(pMixedCtx);
7893 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7894 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7895 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7896 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7897}
7898#endif /* unused */
7899
7900
7901/**
7902 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7903 *
7904 * @param pVCpu The cross context virtual CPU structure.
7905 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7906 * out-of-sync. Make sure to update the required fields
7907 * before using them.
7908 * @param uVector The software interrupt vector number.
7909 * @param cbInstr The value of RIP that is to be pushed on the guest
7910 * stack.
7911 */
7912DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7913{
7914 NOREF(pMixedCtx);
7915 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7916 if ( uVector == X86_XCPT_BP
7917 || uVector == X86_XCPT_OF)
7918 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7919 else
7920 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7921 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7922}
7923
7924
7925/**
7926 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7927 * stack.
7928 *
7929 * @returns Strict VBox status code (i.e. informational status codes too).
7930 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7931 * @param pVM The cross context VM structure.
7932 * @param pMixedCtx Pointer to the guest-CPU context.
7933 * @param uValue The value to push to the guest stack.
7934 */
7935DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7936{
7937 /*
7938 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7939 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7940 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7941 */
7942 if (pMixedCtx->sp == 1)
7943 return VINF_EM_RESET;
7944 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7945 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7946 AssertRC(rc);
7947 return rc;
7948}
7949
7950
7951/**
7952 * Injects an event into the guest upon VM-entry by updating the relevant fields
7953 * in the VM-entry area in the VMCS.
7954 *
7955 * @returns Strict VBox status code (i.e. informational status codes too).
7956 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7957 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7958 *
7959 * @param pVCpu The cross context virtual CPU structure.
7960 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7961 * be out-of-sync. Make sure to update the required
7962 * fields before using them.
7963 * @param u64IntInfo The VM-entry interruption-information field.
7964 * @param cbInstr The VM-entry instruction length in bytes (for
7965 * software interrupts, exceptions and privileged
7966 * software exceptions).
7967 * @param u32ErrCode The VM-entry exception error code.
7968 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7969 * @param puIntrState Pointer to the current guest interruptibility-state.
7970 * This interruptibility-state will be updated if
7971 * necessary. This cannot not be NULL.
7972 * @param fStepping Whether we're running in
7973 * hmR0VmxRunGuestCodeStep() and should return
7974 * VINF_EM_DBG_STEPPED if the event is injected
7975 * directly (register modified by us, not by
7976 * hardware on VM-entry).
7977 *
7978 * @remarks Requires CR0!
7979 * @remarks No-long-jump zone!!!
7980 */
7981static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7982 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7983 uint32_t *puIntrState)
7984{
7985 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7986 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7987 Assert(puIntrState);
7988 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7989
7990 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7991 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7992
7993#ifdef VBOX_STRICT
7994 /* Validate the error-code-valid bit for hardware exceptions. */
7995 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7996 {
7997 switch (uVector)
7998 {
7999 case X86_XCPT_PF:
8000 case X86_XCPT_DF:
8001 case X86_XCPT_TS:
8002 case X86_XCPT_NP:
8003 case X86_XCPT_SS:
8004 case X86_XCPT_GP:
8005 case X86_XCPT_AC:
8006 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8007 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8008 /* fallthru */
8009 default:
8010 break;
8011 }
8012 }
8013#endif
8014
8015 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8016 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8017 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8018
8019 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8020
8021 /* We require CR0 to check if the guest is in real-mode. */
8022 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8023 AssertRCReturn(rc, rc);
8024
8025 /*
8026 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8027 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8028 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8029 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8030 */
8031 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8032 {
8033 PVM pVM = pVCpu->CTX_SUFF(pVM);
8034 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8035 {
8036 Assert(PDMVmmDevHeapIsEnabled(pVM));
8037 Assert(pVM->hm.s.vmx.pRealModeTSS);
8038
8039 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8040 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8041 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8042 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8043 AssertRCReturn(rc, rc);
8044 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8045
8046 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8047 size_t const cbIdtEntry = sizeof(X86IDTR16);
8048 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8049 {
8050 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8051 if (uVector == X86_XCPT_DF)
8052 return VINF_EM_RESET;
8053
8054 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8055 if (uVector == X86_XCPT_GP)
8056 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8057
8058 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8059 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8060 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8061 fStepping, puIntrState);
8062 }
8063
8064 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8065 uint16_t uGuestIp = pMixedCtx->ip;
8066 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8067 {
8068 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8069 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8070 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8071 }
8072 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8073 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8074
8075 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8076 X86IDTR16 IdtEntry;
8077 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8078 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8079 AssertRCReturn(rc, rc);
8080
8081 /* Construct the stack frame for the interrupt/exception handler. */
8082 VBOXSTRICTRC rcStrict;
8083 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8084 if (rcStrict == VINF_SUCCESS)
8085 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8086 if (rcStrict == VINF_SUCCESS)
8087 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8088
8089 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8090 if (rcStrict == VINF_SUCCESS)
8091 {
8092 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8093 pMixedCtx->rip = IdtEntry.offSel;
8094 pMixedCtx->cs.Sel = IdtEntry.uSel;
8095 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8096 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8097 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8098 && uVector == X86_XCPT_PF)
8099 pMixedCtx->cr2 = GCPtrFaultAddress;
8100
8101 /* If any other guest-state bits are changed here, make sure to update
8102 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8103 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8104 | HM_CHANGED_GUEST_RIP
8105 | HM_CHANGED_GUEST_RFLAGS
8106 | HM_CHANGED_GUEST_RSP);
8107
8108 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8109 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8110 {
8111 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8112 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8113 Log4(("Clearing inhibition due to STI.\n"));
8114 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8115 }
8116 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8117 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8118
8119 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8120 it, if we are returning to ring-3 before executing guest code. */
8121 pVCpu->hm.s.Event.fPending = false;
8122
8123 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8124 if (fStepping)
8125 rcStrict = VINF_EM_DBG_STEPPED;
8126 }
8127 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8128 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8129 return rcStrict;
8130 }
8131
8132 /*
8133 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8134 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8135 */
8136 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8137 }
8138
8139 /* Validate. */
8140 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8141 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8142 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8143
8144 /* Inject. */
8145 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8146 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8147 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8148 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8149
8150 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8151 && uVector == X86_XCPT_PF)
8152 pMixedCtx->cr2 = GCPtrFaultAddress;
8153
8154 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8155 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8156
8157 AssertRCReturn(rc, rc);
8158 return VINF_SUCCESS;
8159}
8160
8161
8162/**
8163 * Clears the interrupt-window exiting control in the VMCS and if necessary
8164 * clears the current event in the VMCS as well.
8165 *
8166 * @returns VBox status code.
8167 * @param pVCpu The cross context virtual CPU structure.
8168 *
8169 * @remarks Use this function only to clear events that have not yet been
8170 * delivered to the guest but are injected in the VMCS!
8171 * @remarks No-long-jump zone!!!
8172 */
8173static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8174{
8175 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8176
8177 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8178 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8179
8180 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8181 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8182}
8183
8184
8185/**
8186 * Enters the VT-x session.
8187 *
8188 * @returns VBox status code.
8189 * @param pVM The cross context VM structure.
8190 * @param pVCpu The cross context virtual CPU structure.
8191 * @param pCpu Pointer to the CPU info struct.
8192 */
8193VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8194{
8195 AssertPtr(pVM);
8196 AssertPtr(pVCpu);
8197 Assert(pVM->hm.s.vmx.fSupported);
8198 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8199 NOREF(pCpu); NOREF(pVM);
8200
8201 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8202 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8203
8204#ifdef VBOX_STRICT
8205 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8206 RTCCUINTREG uHostCR4 = ASMGetCR4();
8207 if (!(uHostCR4 & X86_CR4_VMXE))
8208 {
8209 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8210 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8211 }
8212#endif
8213
8214 /*
8215 * Load the VCPU's VMCS as the current (and active) one.
8216 */
8217 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8218 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8219 if (RT_FAILURE(rc))
8220 return rc;
8221
8222 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8223 pVCpu->hm.s.fLeaveDone = false;
8224 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8225
8226 return VINF_SUCCESS;
8227}
8228
8229
8230/**
8231 * The thread-context callback (only on platforms which support it).
8232 *
8233 * @param enmEvent The thread-context event.
8234 * @param pVCpu The cross context virtual CPU structure.
8235 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8236 * @thread EMT(pVCpu)
8237 */
8238VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8239{
8240 NOREF(fGlobalInit);
8241
8242 switch (enmEvent)
8243 {
8244 case RTTHREADCTXEVENT_OUT:
8245 {
8246 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8247 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8248 VMCPU_ASSERT_EMT(pVCpu);
8249
8250 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8251
8252 /* No longjmps (logger flushes, locks) in this fragile context. */
8253 VMMRZCallRing3Disable(pVCpu);
8254 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8255
8256 /*
8257 * Restore host-state (FPU, debug etc.)
8258 */
8259 if (!pVCpu->hm.s.fLeaveDone)
8260 {
8261 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8262 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8263 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8264 pVCpu->hm.s.fLeaveDone = true;
8265 }
8266
8267 /* Leave HM context, takes care of local init (term). */
8268 int rc = HMR0LeaveCpu(pVCpu);
8269 AssertRC(rc); NOREF(rc);
8270
8271 /* Restore longjmp state. */
8272 VMMRZCallRing3Enable(pVCpu);
8273 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8274 break;
8275 }
8276
8277 case RTTHREADCTXEVENT_IN:
8278 {
8279 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8280 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8281 VMCPU_ASSERT_EMT(pVCpu);
8282
8283 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8284 VMMRZCallRing3Disable(pVCpu);
8285 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8286
8287 /* Initialize the bare minimum state required for HM. This takes care of
8288 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8289 int rc = HMR0EnterCpu(pVCpu);
8290 AssertRC(rc);
8291 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8292
8293 /* Load the active VMCS as the current one. */
8294 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8295 {
8296 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8297 AssertRC(rc); NOREF(rc);
8298 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8299 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8300 }
8301 pVCpu->hm.s.fLeaveDone = false;
8302
8303 /* Restore longjmp state. */
8304 VMMRZCallRing3Enable(pVCpu);
8305 break;
8306 }
8307
8308 default:
8309 break;
8310 }
8311}
8312
8313
8314/**
8315 * Saves the host state in the VMCS host-state.
8316 * Sets up the VM-exit MSR-load area.
8317 *
8318 * The CPU state will be loaded from these fields on every successful VM-exit.
8319 *
8320 * @returns VBox status code.
8321 * @param pVM The cross context VM structure.
8322 * @param pVCpu The cross context virtual CPU structure.
8323 *
8324 * @remarks No-long-jump zone!!!
8325 */
8326static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8327{
8328 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8329
8330 int rc = VINF_SUCCESS;
8331 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8332 {
8333 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8334 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8335
8336 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8337 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8338
8339 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8340 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8341
8342 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8343 }
8344 return rc;
8345}
8346
8347
8348/**
8349 * Saves the host state in the VMCS host-state.
8350 *
8351 * @returns VBox status code.
8352 * @param pVM The cross context VM structure.
8353 * @param pVCpu The cross context virtual CPU structure.
8354 *
8355 * @remarks No-long-jump zone!!!
8356 */
8357VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8358{
8359 AssertPtr(pVM);
8360 AssertPtr(pVCpu);
8361
8362 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8363
8364 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8365 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8366 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8367 return hmR0VmxSaveHostState(pVM, pVCpu);
8368}
8369
8370
8371/**
8372 * Loads the guest state into the VMCS guest-state area.
8373 *
8374 * The will typically be done before VM-entry when the guest-CPU state and the
8375 * VMCS state may potentially be out of sync.
8376 *
8377 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8378 * VM-entry controls.
8379 * Sets up the appropriate VMX non-root function to execute guest code based on
8380 * the guest CPU mode.
8381 *
8382 * @returns VBox strict status code.
8383 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8384 * without unrestricted guest access and the VMMDev is not presently
8385 * mapped (e.g. EFI32).
8386 *
8387 * @param pVM The cross context VM structure.
8388 * @param pVCpu The cross context virtual CPU structure.
8389 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8390 * out-of-sync. Make sure to update the required fields
8391 * before using them.
8392 *
8393 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8394 * caller disables then again on successfull return. Confusing.)
8395 */
8396static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8397{
8398 AssertPtr(pVM);
8399 AssertPtr(pVCpu);
8400 AssertPtr(pMixedCtx);
8401 HMVMX_ASSERT_PREEMPT_SAFE();
8402
8403 VMMRZCallRing3Disable(pVCpu);
8404 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8405
8406 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8407
8408 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8409
8410 /* Determine real-on-v86 mode. */
8411 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8412 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8413 && CPUMIsGuestInRealModeEx(pMixedCtx))
8414 {
8415 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8416 }
8417
8418 /*
8419 * Load the guest-state into the VMCS.
8420 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8421 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8422 */
8423 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8424 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8425
8426 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8427 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8428 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8429
8430 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8431 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8432 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8433
8434 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8435 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8436
8437 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8438 if (rcStrict == VINF_SUCCESS)
8439 { /* likely */ }
8440 else
8441 {
8442 VMMRZCallRing3Enable(pVCpu);
8443 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8444 return rcStrict;
8445 }
8446
8447 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8448 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8449 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8450
8451 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8452 determine we don't have to swap EFER after all. */
8453 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8454 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8455
8456 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8457 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8458
8459 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8460 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8461
8462 /*
8463 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8464 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8465 */
8466 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8467 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8468
8469 /* Clear any unused and reserved bits. */
8470 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8471
8472 VMMRZCallRing3Enable(pVCpu);
8473
8474 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8475 return rc;
8476}
8477
8478
8479/**
8480 * Loads the state shared between the host and guest into the VMCS.
8481 *
8482 * @param pVM The cross context VM structure.
8483 * @param pVCpu The cross context virtual CPU structure.
8484 * @param pCtx Pointer to the guest-CPU context.
8485 *
8486 * @remarks No-long-jump zone!!!
8487 */
8488static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8489{
8490 NOREF(pVM);
8491
8492 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8493 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8494
8495 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8496 {
8497 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8498 AssertRC(rc);
8499 }
8500
8501 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8502 {
8503 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8504 AssertRC(rc);
8505
8506 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8507 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8508 {
8509 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8510 AssertRC(rc);
8511 }
8512 }
8513
8514 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8515 {
8516 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8517 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8518 }
8519
8520 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8521 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8522 {
8523 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8524 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8525 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8526 AssertRC(rc);
8527 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8528 }
8529
8530 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8531 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8532}
8533
8534
8535/**
8536 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8537 *
8538 * @returns Strict VBox status code (i.e. informational status codes too).
8539 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8540 * without unrestricted guest access and the VMMDev is not presently
8541 * mapped (e.g. EFI32).
8542 *
8543 * @param pVM The cross context VM structure.
8544 * @param pVCpu The cross context virtual CPU structure.
8545 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8546 * out-of-sync. Make sure to update the required fields
8547 * before using them.
8548 */
8549static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8550{
8551 HMVMX_ASSERT_PREEMPT_SAFE();
8552
8553 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8554#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8555 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8556#endif
8557
8558 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8559 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8560 {
8561 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8562 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8563 { /* likely */}
8564 else
8565 {
8566 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8567 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8568 }
8569 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8570 }
8571 else if (HMCPU_CF_VALUE(pVCpu))
8572 {
8573 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8574 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8575 { /* likely */}
8576 else
8577 {
8578 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8579 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8580 return rcStrict;
8581 }
8582 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8583 }
8584
8585 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8586 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8587 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8588 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8589 return rcStrict;
8590}
8591
8592
8593/**
8594 * Does the preparations before executing guest code in VT-x.
8595 *
8596 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8597 * recompiler/IEM. We must be cautious what we do here regarding committing
8598 * guest-state information into the VMCS assuming we assuredly execute the
8599 * guest in VT-x mode.
8600 *
8601 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8602 * the common-state (TRPM/forceflags), we must undo those changes so that the
8603 * recompiler/IEM can (and should) use them when it resumes guest execution.
8604 * Otherwise such operations must be done when we can no longer exit to ring-3.
8605 *
8606 * @returns Strict VBox status code (i.e. informational status codes too).
8607 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8608 * have been disabled.
8609 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8610 * double-fault into the guest.
8611 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8612 * dispatched directly.
8613 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8614 *
8615 * @param pVM The cross context VM structure.
8616 * @param pVCpu The cross context virtual CPU structure.
8617 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8618 * out-of-sync. Make sure to update the required fields
8619 * before using them.
8620 * @param pVmxTransient Pointer to the VMX transient structure.
8621 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8622 * us ignore some of the reasons for returning to
8623 * ring-3, and return VINF_EM_DBG_STEPPED if event
8624 * dispatching took place.
8625 */
8626static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8627{
8628 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8629
8630#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8631 PGMRZDynMapFlushAutoSet(pVCpu);
8632#endif
8633
8634 /* Check force flag actions that might require us to go back to ring-3. */
8635 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8636 if (rcStrict == VINF_SUCCESS)
8637 { /* FFs doesn't get set all the time. */ }
8638 else
8639 return rcStrict;
8640
8641 if (TRPMHasTrap(pVCpu))
8642 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8643 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8644
8645 /*
8646 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8647 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8648 */
8649 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8650 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8651 { /* likely */ }
8652 else
8653 {
8654 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8655 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8656 return rcStrict;
8657 }
8658
8659 /*
8660 * Load the guest state bits, we can handle longjmps/getting preempted here.
8661 *
8662 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8663 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8664 * Hence, this needs to be done -after- injection of events.
8665 */
8666 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8667 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8668 { /* likely */ }
8669 else
8670 return rcStrict;
8671
8672 /*
8673 * No longjmps to ring-3 from this point on!!!
8674 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8675 * This also disables flushing of the R0-logger instance (if any).
8676 */
8677 VMMRZCallRing3Disable(pVCpu);
8678
8679 /*
8680 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8681 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8682 *
8683 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8684 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8685 *
8686 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8687 * executing guest code.
8688 */
8689 pVmxTransient->fEFlags = ASMIntDisableFlags();
8690
8691 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8692 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8693 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8694 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8695 {
8696 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8697 {
8698 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8699 pVCpu->hm.s.Event.fPending = false;
8700
8701 return VINF_SUCCESS;
8702 }
8703
8704 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8705 rcStrict = VINF_EM_RAW_INTERRUPT;
8706 }
8707 else
8708 {
8709 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8710 rcStrict = VINF_EM_RAW_TO_R3;
8711 }
8712
8713 ASMSetFlags(pVmxTransient->fEFlags);
8714 VMMRZCallRing3Enable(pVCpu);
8715
8716 return rcStrict;
8717}
8718
8719
8720/**
8721 * Prepares to run guest code in VT-x and we've committed to doing so. This
8722 * means there is no backing out to ring-3 or anywhere else at this
8723 * point.
8724 *
8725 * @param pVM The cross context VM structure.
8726 * @param pVCpu The cross context virtual CPU structure.
8727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8728 * out-of-sync. Make sure to update the required fields
8729 * before using them.
8730 * @param pVmxTransient Pointer to the VMX transient structure.
8731 *
8732 * @remarks Called with preemption disabled.
8733 * @remarks No-long-jump zone!!!
8734 */
8735static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8736{
8737 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8738 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8739 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8740
8741 /*
8742 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8743 */
8744 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8745 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8746
8747#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8748 if (!CPUMIsGuestFPUStateActive(pVCpu))
8749 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8750 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8751 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8752#endif
8753
8754 if ( pVCpu->hm.s.fPreloadGuestFpu
8755 && !CPUMIsGuestFPUStateActive(pVCpu))
8756 {
8757 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8758 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8759 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8760 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8761 }
8762
8763 /*
8764 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8765 */
8766 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8767 && pVCpu->hm.s.vmx.cMsrs > 0)
8768 {
8769 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8770 }
8771
8772 /*
8773 * Load the host state bits as we may've been preempted (only happens when
8774 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8775 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8776 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8777 * See @bugref{8432}.
8778 */
8779 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8780 {
8781 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8782 AssertRC(rc);
8783 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8784 }
8785 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8786
8787 /*
8788 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8789 */
8790 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8791 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8792 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8793
8794 /* Store status of the shared guest-host state at the time of VM-entry. */
8795#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8796 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8797 {
8798 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8799 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8800 }
8801 else
8802#endif
8803 {
8804 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8805 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8806 }
8807 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8808
8809 /*
8810 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8811 */
8812 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8813 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8814
8815 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8816 RTCPUID idCurrentCpu = pCpu->idCpu;
8817 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8818 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8819 {
8820 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8821 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8822 }
8823
8824 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8825 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8826 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8827 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8828
8829 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8830
8831 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8832 to start executing. */
8833
8834 /*
8835 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8836 */
8837 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8838 {
8839 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8840 {
8841 bool fMsrUpdated;
8842 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8843 AssertRC(rc2);
8844 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8845
8846 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8847 &fMsrUpdated);
8848 AssertRC(rc2);
8849 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8850
8851 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8852 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8853 }
8854 else
8855 {
8856 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8857 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8858 }
8859 }
8860
8861#ifdef VBOX_STRICT
8862 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8863 hmR0VmxCheckHostEferMsr(pVCpu);
8864 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8865#endif
8866#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8867 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8868 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8869 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8870#endif
8871}
8872
8873
8874/**
8875 * Performs some essential restoration of state after running guest code in
8876 * VT-x.
8877 *
8878 * @param pVM The cross context VM structure.
8879 * @param pVCpu The cross context virtual CPU structure.
8880 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8881 * out-of-sync. Make sure to update the required fields
8882 * before using them.
8883 * @param pVmxTransient Pointer to the VMX transient structure.
8884 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8885 *
8886 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8887 *
8888 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8889 * unconditionally when it is safe to do so.
8890 */
8891static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8892{
8893 NOREF(pVM);
8894
8895 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8896
8897 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8898 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8899 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8900 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8901 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8902 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8903
8904 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8905 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8906
8907 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8908 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8909 Assert(!ASMIntAreEnabled());
8910 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8911
8912#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8913 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8914 {
8915 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8916 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8917 }
8918#endif
8919
8920#if HC_ARCH_BITS == 64
8921 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8922#endif
8923#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8924 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8925 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8926 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8927#else
8928 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8929#endif
8930#ifdef VBOX_STRICT
8931 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8932#endif
8933 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8934 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8935
8936 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8937 uint32_t uExitReason;
8938 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8939 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8940 AssertRC(rc);
8941 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8942 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8943
8944 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8945 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8946 {
8947 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8948 pVmxTransient->fVMEntryFailed));
8949 return;
8950 }
8951
8952 /*
8953 * Update the VM-exit history array here even if the VM-entry failed due to:
8954 * - Invalid guest state.
8955 * - MSR loading.
8956 * - Machine-check event.
8957 *
8958 * In any of the above cases we will still have a "valid" VM-exit reason
8959 * despite @a fVMEntryFailed being false.
8960 *
8961 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8962 */
8963 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8964
8965 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8966 {
8967 /** @todo We can optimize this by only syncing with our force-flags when
8968 * really needed and keeping the VMCS state as it is for most
8969 * VM-exits. */
8970 /* Update the guest interruptibility-state from the VMCS. */
8971 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8972
8973#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8974 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8975 AssertRC(rc);
8976#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8977 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8978 AssertRC(rc);
8979#endif
8980
8981 /*
8982 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8983 * we eventually get a VM-exit for any reason.
8984 *
8985 * This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is why it's done here as it's easier and
8986 * no less efficient to deal with it here than making hmR0VmxSaveGuestState() cope with longjmps safely
8987 * (see VMCPU_FF_HM_UPDATE_CR3 handling).
8988 */
8989 /** @todo r=ramshankar: The 2nd para in the above comment is
8990 * outdated, we no longer longjmp to ring-3 on setting
8991 * the TPR, but regardless we can probably rework this
8992 * portion of the code a bit. */
8993 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8994 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8995 {
8996 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8997 AssertRC(rc);
8998 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8999 }
9000 }
9001}
9002
9003
9004/**
9005 * Runs the guest code using VT-x the normal way.
9006 *
9007 * @returns VBox status code.
9008 * @param pVM The cross context VM structure.
9009 * @param pVCpu The cross context virtual CPU structure.
9010 * @param pCtx Pointer to the guest-CPU context.
9011 *
9012 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9013 */
9014static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9015{
9016 VMXTRANSIENT VmxTransient;
9017 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9018 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9019 uint32_t cLoops = 0;
9020
9021 for (;; cLoops++)
9022 {
9023 Assert(!HMR0SuspendPending());
9024 HMVMX_ASSERT_CPU_SAFE();
9025
9026 /* Preparatory work for running guest code, this may force us to return
9027 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9028 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9029 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9030 if (rcStrict != VINF_SUCCESS)
9031 break;
9032
9033 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9034 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9035 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9036
9037 /* Restore any residual host-state and save any bits shared between host
9038 and guest into the guest-CPU state. Re-enables interrupts! */
9039 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9040
9041 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9042 if (RT_SUCCESS(rcRun))
9043 { /* very likely */ }
9044 else
9045 {
9046 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9047 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9048 return rcRun;
9049 }
9050
9051 /* Profile the VM-exit. */
9052 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9054 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9055 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9056 HMVMX_START_EXIT_DISPATCH_PROF();
9057
9058 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9059
9060 /* Handle the VM-exit. */
9061#ifdef HMVMX_USE_FUNCTION_TABLE
9062 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9063#else
9064 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9065#endif
9066 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9067 if (rcStrict == VINF_SUCCESS)
9068 {
9069 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9070 continue; /* likely */
9071 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9072 rcStrict = VINF_EM_RAW_INTERRUPT;
9073 }
9074 break;
9075 }
9076
9077 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9078 return rcStrict;
9079}
9080
9081
9082
9083/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9084 * probes.
9085 *
9086 * The following few functions and associated structure contains the bloat
9087 * necessary for providing detailed debug events and dtrace probes as well as
9088 * reliable host side single stepping. This works on the principle of
9089 * "subclassing" the normal execution loop and workers. We replace the loop
9090 * method completely and override selected helpers to add necessary adjustments
9091 * to their core operation.
9092 *
9093 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9094 * any performance for debug and analysis features.
9095 *
9096 * @{
9097 */
9098
9099/**
9100 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9101 * the debug run loop.
9102 */
9103typedef struct VMXRUNDBGSTATE
9104{
9105 /** The RIP we started executing at. This is for detecting that we stepped. */
9106 uint64_t uRipStart;
9107 /** The CS we started executing with. */
9108 uint16_t uCsStart;
9109
9110 /** Whether we've actually modified the 1st execution control field. */
9111 bool fModifiedProcCtls : 1;
9112 /** Whether we've actually modified the 2nd execution control field. */
9113 bool fModifiedProcCtls2 : 1;
9114 /** Whether we've actually modified the exception bitmap. */
9115 bool fModifiedXcptBitmap : 1;
9116
9117 /** We desire the modified the CR0 mask to be cleared. */
9118 bool fClearCr0Mask : 1;
9119 /** We desire the modified the CR4 mask to be cleared. */
9120 bool fClearCr4Mask : 1;
9121 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9122 uint32_t fCpe1Extra;
9123 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9124 uint32_t fCpe1Unwanted;
9125 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9126 uint32_t fCpe2Extra;
9127 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9128 uint32_t bmXcptExtra;
9129 /** The sequence number of the Dtrace provider settings the state was
9130 * configured against. */
9131 uint32_t uDtraceSettingsSeqNo;
9132 /** VM-exits to check (one bit per VM-exit). */
9133 uint32_t bmExitsToCheck[3];
9134
9135 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9136 uint32_t fProcCtlsInitial;
9137 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9138 uint32_t fProcCtls2Initial;
9139 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9140 uint32_t bmXcptInitial;
9141} VMXRUNDBGSTATE;
9142AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9143typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9144
9145
9146/**
9147 * Initializes the VMXRUNDBGSTATE structure.
9148 *
9149 * @param pVCpu The cross context virtual CPU structure of the
9150 * calling EMT.
9151 * @param pCtx The CPU register context to go with @a pVCpu.
9152 * @param pDbgState The structure to initialize.
9153 */
9154DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9155{
9156 pDbgState->uRipStart = pCtx->rip;
9157 pDbgState->uCsStart = pCtx->cs.Sel;
9158
9159 pDbgState->fModifiedProcCtls = false;
9160 pDbgState->fModifiedProcCtls2 = false;
9161 pDbgState->fModifiedXcptBitmap = false;
9162 pDbgState->fClearCr0Mask = false;
9163 pDbgState->fClearCr4Mask = false;
9164 pDbgState->fCpe1Extra = 0;
9165 pDbgState->fCpe1Unwanted = 0;
9166 pDbgState->fCpe2Extra = 0;
9167 pDbgState->bmXcptExtra = 0;
9168 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9169 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9170 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9171}
9172
9173
9174/**
9175 * Updates the VMSC fields with changes requested by @a pDbgState.
9176 *
9177 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9178 * immediately before executing guest code, i.e. when interrupts are disabled.
9179 * We don't check status codes here as we cannot easily assert or return in the
9180 * latter case.
9181 *
9182 * @param pVCpu The cross context virtual CPU structure.
9183 * @param pDbgState The debug state.
9184 */
9185DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9186{
9187 /*
9188 * Ensure desired flags in VMCS control fields are set.
9189 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9190 *
9191 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9192 * there should be no stale data in pCtx at this point.
9193 */
9194 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9195 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9196 {
9197 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9198 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9199 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9200 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9201 pDbgState->fModifiedProcCtls = true;
9202 }
9203
9204 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9205 {
9206 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9207 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9208 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9209 pDbgState->fModifiedProcCtls2 = true;
9210 }
9211
9212 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9213 {
9214 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9215 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9216 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9217 pDbgState->fModifiedXcptBitmap = true;
9218 }
9219
9220 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9221 {
9222 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9223 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9224 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9225 }
9226
9227 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9228 {
9229 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9230 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9231 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9232 }
9233}
9234
9235
9236DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9237{
9238 /*
9239 * Restore VM-exit control settings as we may not reenter this function the
9240 * next time around.
9241 */
9242 /* We reload the initial value, trigger what we can of recalculations the
9243 next time around. From the looks of things, that's all that's required atm. */
9244 if (pDbgState->fModifiedProcCtls)
9245 {
9246 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9247 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9248 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9249 AssertRCReturn(rc2, rc2);
9250 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9251 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9252 }
9253
9254 /* We're currently the only ones messing with this one, so just restore the
9255 cached value and reload the field. */
9256 if ( pDbgState->fModifiedProcCtls2
9257 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9258 {
9259 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9260 AssertRCReturn(rc2, rc2);
9261 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9262 }
9263
9264 /* If we've modified the exception bitmap, we restore it and trigger
9265 reloading and partial recalculation the next time around. */
9266 if (pDbgState->fModifiedXcptBitmap)
9267 {
9268 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9269 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9270 }
9271
9272 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9273 if (pDbgState->fClearCr0Mask)
9274 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9275
9276 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9277 if (pDbgState->fClearCr4Mask)
9278 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9279
9280 return rcStrict;
9281}
9282
9283
9284/**
9285 * Configures VM-exit controls for current DBGF and DTrace settings.
9286 *
9287 * This updates @a pDbgState and the VMCS execution control fields to reflect
9288 * the necessary VM-exits demanded by DBGF and DTrace.
9289 *
9290 * @param pVM The cross context VM structure.
9291 * @param pVCpu The cross context virtual CPU structure.
9292 * @param pCtx Pointer to the guest-CPU context.
9293 * @param pDbgState The debug state.
9294 * @param pVmxTransient Pointer to the VMX transient structure. May update
9295 * fUpdateTscOffsettingAndPreemptTimer.
9296 */
9297static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9298 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9299{
9300 /*
9301 * Take down the dtrace serial number so we can spot changes.
9302 */
9303 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9304 ASMCompilerBarrier();
9305
9306 /*
9307 * We'll rebuild most of the middle block of data members (holding the
9308 * current settings) as we go along here, so start by clearing it all.
9309 */
9310 pDbgState->bmXcptExtra = 0;
9311 pDbgState->fCpe1Extra = 0;
9312 pDbgState->fCpe1Unwanted = 0;
9313 pDbgState->fCpe2Extra = 0;
9314 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9315 pDbgState->bmExitsToCheck[i] = 0;
9316
9317 /*
9318 * Software interrupts (INT XXh) - no idea how to trigger these...
9319 */
9320 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9321 || VBOXVMM_INT_SOFTWARE_ENABLED())
9322 {
9323 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9324 }
9325
9326 /*
9327 * INT3 breakpoints - triggered by #BP exceptions.
9328 */
9329 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9330 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9331
9332 /*
9333 * Exception bitmap and XCPT events+probes.
9334 */
9335 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9336 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9337 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9338
9339 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9340 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9341 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9342 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9343 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9344 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9345 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9346 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9347 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9348 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9349 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9350 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9351 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9352 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9353 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9354 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9355 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9356 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9357
9358 if (pDbgState->bmXcptExtra)
9359 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9360
9361 /*
9362 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9363 *
9364 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9365 * So, when adding/changing/removing please don't forget to update it.
9366 *
9367 * Some of the macros are picking up local variables to save horizontal space,
9368 * (being able to see it in a table is the lesser evil here).
9369 */
9370#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9371 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9372 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9373#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9374 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9375 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9376 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9377 } else do { } while (0)
9378#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9379 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9380 { \
9381 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9382 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9383 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9384 } else do { } while (0)
9385#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9386 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9387 { \
9388 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9389 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9390 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9391 } else do { } while (0)
9392#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9393 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9394 { \
9395 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9396 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9397 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9398 } else do { } while (0)
9399
9400 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9401 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9402 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9403 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9404 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9405
9406 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9408 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9410 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9414 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9416 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9418 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9420 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9422 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9423 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9424 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9426 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9427 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9428 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9429 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9430 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9431 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9432 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9433 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9434 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9435 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9436 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9437 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9438 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9439 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9440 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9441 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9442
9443 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9444 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9445 {
9446 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9447 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9448 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9449 AssertRC(rc2);
9450
9451#if 0 /** @todo fix me */
9452 pDbgState->fClearCr0Mask = true;
9453 pDbgState->fClearCr4Mask = true;
9454#endif
9455 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9456 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9457 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9458 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9459 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9460 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9461 require clearing here and in the loop if we start using it. */
9462 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9463 }
9464 else
9465 {
9466 if (pDbgState->fClearCr0Mask)
9467 {
9468 pDbgState->fClearCr0Mask = false;
9469 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9470 }
9471 if (pDbgState->fClearCr4Mask)
9472 {
9473 pDbgState->fClearCr4Mask = false;
9474 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9475 }
9476 }
9477 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9478 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9479
9480 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9481 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9482 {
9483 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9484 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9485 }
9486 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9487 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9488
9489 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9490 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9491 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9492 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9493 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9494 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9495 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9496 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9497#if 0 /** @todo too slow, fix handler. */
9498 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9499#endif
9500 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9501
9502 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9503 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9504 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9505 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9506 {
9507 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9508 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9509 }
9510 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9511 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9512 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9513 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9514
9515 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9516 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9517 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9518 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9519 {
9520 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9521 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9522 }
9523 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9524 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9525 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9526 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9527
9528 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9529 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9530 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9531 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9532 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9533 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9534 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9535 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9536 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9537 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9538 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9539 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9540 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9541 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9542 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9543 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9544 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9545 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9546 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9547 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9548 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9549 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9550
9551#undef IS_EITHER_ENABLED
9552#undef SET_ONLY_XBM_IF_EITHER_EN
9553#undef SET_CPE1_XBM_IF_EITHER_EN
9554#undef SET_CPEU_XBM_IF_EITHER_EN
9555#undef SET_CPE2_XBM_IF_EITHER_EN
9556
9557 /*
9558 * Sanitize the control stuff.
9559 */
9560 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9561 if (pDbgState->fCpe2Extra)
9562 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9563 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9564 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9565 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9566 {
9567 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9568 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9569 }
9570
9571 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9572 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9573 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9574 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9575}
9576
9577
9578/**
9579 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9580 * appropriate.
9581 *
9582 * The caller has checked the VM-exit against the
9583 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9584 * already, so we don't have to do that either.
9585 *
9586 * @returns Strict VBox status code (i.e. informational status codes too).
9587 * @param pVM The cross context VM structure.
9588 * @param pVCpu The cross context virtual CPU structure.
9589 * @param pMixedCtx Pointer to the guest-CPU context.
9590 * @param pVmxTransient Pointer to the VMX-transient structure.
9591 * @param uExitReason The VM-exit reason.
9592 *
9593 * @remarks The name of this function is displayed by dtrace, so keep it short
9594 * and to the point. No longer than 33 chars long, please.
9595 */
9596static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9597 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9598{
9599 /*
9600 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9601 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9602 *
9603 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9604 * does. Must add/change/remove both places. Same ordering, please.
9605 *
9606 * Added/removed events must also be reflected in the next section
9607 * where we dispatch dtrace events.
9608 */
9609 bool fDtrace1 = false;
9610 bool fDtrace2 = false;
9611 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9612 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9613 uint32_t uEventArg = 0;
9614#define SET_EXIT(a_EventSubName) \
9615 do { \
9616 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9617 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9618 } while (0)
9619#define SET_BOTH(a_EventSubName) \
9620 do { \
9621 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9622 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9623 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9624 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9625 } while (0)
9626 switch (uExitReason)
9627 {
9628 case VMX_EXIT_MTF:
9629 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9630
9631 case VMX_EXIT_XCPT_OR_NMI:
9632 {
9633 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9634 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9635 {
9636 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9637 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9638 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9639 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9640 {
9641 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9642 {
9643 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9644 uEventArg = pVmxTransient->uExitIntErrorCode;
9645 }
9646 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9647 switch (enmEvent1)
9648 {
9649 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9650 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9651 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9652 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9653 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9654 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9655 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9656 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9657 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9658 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9659 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9660 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9661 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9662 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9663 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9664 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9665 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9666 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9667 default: break;
9668 }
9669 }
9670 else
9671 AssertFailed();
9672 break;
9673
9674 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9675 uEventArg = idxVector;
9676 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9677 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9678 break;
9679 }
9680 break;
9681 }
9682
9683 case VMX_EXIT_TRIPLE_FAULT:
9684 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9685 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9686 break;
9687 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9688 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9689 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9690 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9691 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9692
9693 /* Instruction specific VM-exits: */
9694 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9695 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9696 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9697 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9698 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9699 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9700 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9701 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9702 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9703 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9704 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9705 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9706 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9707 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9708 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9709 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9710 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9711 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9712 case VMX_EXIT_MOV_CRX:
9713 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9714/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9715* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9716 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9717 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9718 SET_BOTH(CRX_READ);
9719 else
9720 SET_BOTH(CRX_WRITE);
9721 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9722 break;
9723 case VMX_EXIT_MOV_DRX:
9724 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9725 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9726 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9727 SET_BOTH(DRX_READ);
9728 else
9729 SET_BOTH(DRX_WRITE);
9730 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9731 break;
9732 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9733 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9734 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9735 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9736 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9737 case VMX_EXIT_XDTR_ACCESS:
9738 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9739 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9740 {
9741 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9742 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9743 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9744 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9745 }
9746 break;
9747
9748 case VMX_EXIT_TR_ACCESS:
9749 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9750 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9751 {
9752 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9753 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9754 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9755 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9756 }
9757 break;
9758
9759 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9760 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9761 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9762 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9763 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9764 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9765 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9766 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9767 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9768 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9769 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9770
9771 /* Events that aren't relevant at this point. */
9772 case VMX_EXIT_EXT_INT:
9773 case VMX_EXIT_INT_WINDOW:
9774 case VMX_EXIT_NMI_WINDOW:
9775 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9776 case VMX_EXIT_PREEMPT_TIMER:
9777 case VMX_EXIT_IO_INSTR:
9778 break;
9779
9780 /* Errors and unexpected events. */
9781 case VMX_EXIT_INIT_SIGNAL:
9782 case VMX_EXIT_SIPI:
9783 case VMX_EXIT_IO_SMI:
9784 case VMX_EXIT_SMI:
9785 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9786 case VMX_EXIT_ERR_MSR_LOAD:
9787 case VMX_EXIT_ERR_MACHINE_CHECK:
9788 break;
9789
9790 default:
9791 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9792 break;
9793 }
9794#undef SET_BOTH
9795#undef SET_EXIT
9796
9797 /*
9798 * Dtrace tracepoints go first. We do them here at once so we don't
9799 * have to copy the guest state saving and stuff a few dozen times.
9800 * Down side is that we've got to repeat the switch, though this time
9801 * we use enmEvent since the probes are a subset of what DBGF does.
9802 */
9803 if (fDtrace1 || fDtrace2)
9804 {
9805 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9806 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9807 switch (enmEvent1)
9808 {
9809 /** @todo consider which extra parameters would be helpful for each probe. */
9810 case DBGFEVENT_END: break;
9811 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9813 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9819 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9820 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9821 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9822 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9823 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9824 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9825 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9826 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9827 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9828 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9829 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9830 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9831 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9832 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9833 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9834 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9835 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9836 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9837 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9838 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9839 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9840 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9841 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9842 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9843 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9844 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9845 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9846 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9847 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9848 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9849 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9850 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9851 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9852 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9853 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9854 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9855 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9856 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9857 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9858 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9859 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9860 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9861 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9862 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9863 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9864 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9865 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9866 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9867 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9868 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9869 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9870 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9871 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9872 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9873 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9874 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9875 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9876 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9877 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9878 }
9879 switch (enmEvent2)
9880 {
9881 /** @todo consider which extra parameters would be helpful for each probe. */
9882 case DBGFEVENT_END: break;
9883 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9884 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9885 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9886 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9887 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9888 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9889 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9890 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9891 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9892 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9893 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9894 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9895 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9896 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9897 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9898 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9899 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9900 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9901 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9902 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9903 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9904 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9905 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9906 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9907 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9908 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9909 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9910 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9911 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9912 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9913 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9914 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9915 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9916 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9917 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9918 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9919 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9920 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9921 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9922 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9923 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9924 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9925 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9926 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9927 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9928 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9929 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9930 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9931 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9932 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9933 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9934 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9935 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9936 }
9937 }
9938
9939 /*
9940 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9941 * the DBGF call will do a full check).
9942 *
9943 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9944 * Note! If we have to events, we prioritize the first, i.e. the instruction
9945 * one, in order to avoid event nesting.
9946 */
9947 if ( enmEvent1 != DBGFEVENT_END
9948 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9949 {
9950 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9951 if (rcStrict != VINF_SUCCESS)
9952 return rcStrict;
9953 }
9954 else if ( enmEvent2 != DBGFEVENT_END
9955 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9956 {
9957 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9958 if (rcStrict != VINF_SUCCESS)
9959 return rcStrict;
9960 }
9961
9962 return VINF_SUCCESS;
9963}
9964
9965
9966/**
9967 * Single-stepping VM-exit filtering.
9968 *
9969 * This is preprocessing the VM-exits and deciding whether we've gotten far
9970 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9971 * handling is performed.
9972 *
9973 * @returns Strict VBox status code (i.e. informational status codes too).
9974 * @param pVM The cross context VM structure.
9975 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9976 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9977 * out-of-sync. Make sure to update the required
9978 * fields before using them.
9979 * @param pVmxTransient Pointer to the VMX-transient structure.
9980 * @param uExitReason The VM-exit reason.
9981 * @param pDbgState The debug state.
9982 */
9983DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9984 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9985{
9986 /*
9987 * Expensive (saves context) generic dtrace VM-exit probe.
9988 */
9989 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9990 { /* more likely */ }
9991 else
9992 {
9993 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9994 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9995 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9996 }
9997
9998 /*
9999 * Check for host NMI, just to get that out of the way.
10000 */
10001 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10002 { /* normally likely */ }
10003 else
10004 {
10005 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10006 AssertRCReturn(rc2, rc2);
10007 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10008 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10009 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10010 }
10011
10012 /*
10013 * Check for single stepping event if we're stepping.
10014 */
10015 if (pVCpu->hm.s.fSingleInstruction)
10016 {
10017 switch (uExitReason)
10018 {
10019 case VMX_EXIT_MTF:
10020 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10021
10022 /* Various events: */
10023 case VMX_EXIT_XCPT_OR_NMI:
10024 case VMX_EXIT_EXT_INT:
10025 case VMX_EXIT_TRIPLE_FAULT:
10026 case VMX_EXIT_INT_WINDOW:
10027 case VMX_EXIT_NMI_WINDOW:
10028 case VMX_EXIT_TASK_SWITCH:
10029 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10030 case VMX_EXIT_APIC_ACCESS:
10031 case VMX_EXIT_EPT_VIOLATION:
10032 case VMX_EXIT_EPT_MISCONFIG:
10033 case VMX_EXIT_PREEMPT_TIMER:
10034
10035 /* Instruction specific VM-exits: */
10036 case VMX_EXIT_CPUID:
10037 case VMX_EXIT_GETSEC:
10038 case VMX_EXIT_HLT:
10039 case VMX_EXIT_INVD:
10040 case VMX_EXIT_INVLPG:
10041 case VMX_EXIT_RDPMC:
10042 case VMX_EXIT_RDTSC:
10043 case VMX_EXIT_RSM:
10044 case VMX_EXIT_VMCALL:
10045 case VMX_EXIT_VMCLEAR:
10046 case VMX_EXIT_VMLAUNCH:
10047 case VMX_EXIT_VMPTRLD:
10048 case VMX_EXIT_VMPTRST:
10049 case VMX_EXIT_VMREAD:
10050 case VMX_EXIT_VMRESUME:
10051 case VMX_EXIT_VMWRITE:
10052 case VMX_EXIT_VMXOFF:
10053 case VMX_EXIT_VMXON:
10054 case VMX_EXIT_MOV_CRX:
10055 case VMX_EXIT_MOV_DRX:
10056 case VMX_EXIT_IO_INSTR:
10057 case VMX_EXIT_RDMSR:
10058 case VMX_EXIT_WRMSR:
10059 case VMX_EXIT_MWAIT:
10060 case VMX_EXIT_MONITOR:
10061 case VMX_EXIT_PAUSE:
10062 case VMX_EXIT_XDTR_ACCESS:
10063 case VMX_EXIT_TR_ACCESS:
10064 case VMX_EXIT_INVEPT:
10065 case VMX_EXIT_RDTSCP:
10066 case VMX_EXIT_INVVPID:
10067 case VMX_EXIT_WBINVD:
10068 case VMX_EXIT_XSETBV:
10069 case VMX_EXIT_RDRAND:
10070 case VMX_EXIT_INVPCID:
10071 case VMX_EXIT_VMFUNC:
10072 case VMX_EXIT_RDSEED:
10073 case VMX_EXIT_XSAVES:
10074 case VMX_EXIT_XRSTORS:
10075 {
10076 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10077 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10078 AssertRCReturn(rc2, rc2);
10079 if ( pMixedCtx->rip != pDbgState->uRipStart
10080 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10081 return VINF_EM_DBG_STEPPED;
10082 break;
10083 }
10084
10085 /* Errors and unexpected events: */
10086 case VMX_EXIT_INIT_SIGNAL:
10087 case VMX_EXIT_SIPI:
10088 case VMX_EXIT_IO_SMI:
10089 case VMX_EXIT_SMI:
10090 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10091 case VMX_EXIT_ERR_MSR_LOAD:
10092 case VMX_EXIT_ERR_MACHINE_CHECK:
10093 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10094 break;
10095
10096 default:
10097 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10098 break;
10099 }
10100 }
10101
10102 /*
10103 * Check for debugger event breakpoints and dtrace probes.
10104 */
10105 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10106 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10107 {
10108 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10109 if (rcStrict != VINF_SUCCESS)
10110 return rcStrict;
10111 }
10112
10113 /*
10114 * Normal processing.
10115 */
10116#ifdef HMVMX_USE_FUNCTION_TABLE
10117 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10118#else
10119 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10120#endif
10121}
10122
10123
10124/**
10125 * Single steps guest code using VT-x.
10126 *
10127 * @returns Strict VBox status code (i.e. informational status codes too).
10128 * @param pVM The cross context VM structure.
10129 * @param pVCpu The cross context virtual CPU structure.
10130 * @param pCtx Pointer to the guest-CPU context.
10131 *
10132 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10133 */
10134static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10135{
10136 VMXTRANSIENT VmxTransient;
10137 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10138
10139 /* Set HMCPU indicators. */
10140 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10141 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10142 pVCpu->hm.s.fDebugWantRdTscExit = false;
10143 pVCpu->hm.s.fUsingDebugLoop = true;
10144
10145 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10146 VMXRUNDBGSTATE DbgState;
10147 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10148 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10149
10150 /*
10151 * The loop.
10152 */
10153 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10154 for (uint32_t cLoops = 0; ; cLoops++)
10155 {
10156 Assert(!HMR0SuspendPending());
10157 HMVMX_ASSERT_CPU_SAFE();
10158 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10159
10160 /*
10161 * Preparatory work for running guest code, this may force us to return
10162 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10163 */
10164 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10165 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10166 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10167 if (rcStrict != VINF_SUCCESS)
10168 break;
10169
10170 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10171 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10172
10173 /*
10174 * Now we can run the guest code.
10175 */
10176 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10177
10178 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10179
10180 /*
10181 * Restore any residual host-state and save any bits shared between host
10182 * and guest into the guest-CPU state. Re-enables interrupts!
10183 */
10184 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10185
10186 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10187 if (RT_SUCCESS(rcRun))
10188 { /* very likely */ }
10189 else
10190 {
10191 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10192 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10193 return rcRun;
10194 }
10195
10196 /* Profile the VM-exit. */
10197 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10199 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10200 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10201 HMVMX_START_EXIT_DISPATCH_PROF();
10202
10203 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10204
10205 /*
10206 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10207 */
10208 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10209 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10210 if (rcStrict != VINF_SUCCESS)
10211 break;
10212 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10213 {
10214 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10215 rcStrict = VINF_EM_RAW_INTERRUPT;
10216 break;
10217 }
10218
10219 /*
10220 * Stepping: Did the RIP change, if so, consider it a single step.
10221 * Otherwise, make sure one of the TFs gets set.
10222 */
10223 if (fStepping)
10224 {
10225 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10226 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10227 AssertRCReturn(rc2, rc2);
10228 if ( pCtx->rip != DbgState.uRipStart
10229 || pCtx->cs.Sel != DbgState.uCsStart)
10230 {
10231 rcStrict = VINF_EM_DBG_STEPPED;
10232 break;
10233 }
10234 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10235 }
10236
10237 /*
10238 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10239 */
10240 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10241 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10242 }
10243
10244 /*
10245 * Clear the X86_EFL_TF if necessary.
10246 */
10247 if (pVCpu->hm.s.fClearTrapFlag)
10248 {
10249 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10250 AssertRCReturn(rc2, rc2);
10251 pVCpu->hm.s.fClearTrapFlag = false;
10252 pCtx->eflags.Bits.u1TF = 0;
10253 }
10254 /** @todo there seems to be issues with the resume flag when the monitor trap
10255 * flag is pending without being used. Seen early in bios init when
10256 * accessing APIC page in protected mode. */
10257
10258 /*
10259 * Restore VM-exit control settings as we may not reenter this function the
10260 * next time around.
10261 */
10262 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10263
10264 /* Restore HMCPU indicators. */
10265 pVCpu->hm.s.fUsingDebugLoop = false;
10266 pVCpu->hm.s.fDebugWantRdTscExit = false;
10267 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10268
10269 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10270 return rcStrict;
10271}
10272
10273
10274/** @} */
10275
10276
10277/**
10278 * Checks if any expensive dtrace probes are enabled and we should go to the
10279 * debug loop.
10280 *
10281 * @returns true if we should use debug loop, false if not.
10282 */
10283static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10284{
10285 /* It's probably faster to OR the raw 32-bit counter variables together.
10286 Since the variables are in an array and the probes are next to one
10287 another (more or less), we have good locality. So, better read
10288 eight-nine cache lines ever time and only have one conditional, than
10289 128+ conditionals, right? */
10290 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10291 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10292 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10293 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10294 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10295 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10296 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10297 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10298 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10299 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10300 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10301 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10302 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10303 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10304 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10305 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10306 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10307 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10308 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10309 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10310 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10311 ) != 0
10312 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10313 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10314 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10315 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10316 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10317 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10318 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10319 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10320 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10321 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10322 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10323 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10324 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10325 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10326 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10327 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10328 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10329 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10330 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10331 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10332 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10333 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10334 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10335 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10336 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10337 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10338 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10339 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10340 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10341 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10342 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10343 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10344 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10345 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10346 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10347 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10348 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10349 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10350 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10351 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10352 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10353 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10354 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10355 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10356 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10357 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10358 ) != 0
10359 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10360 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10361 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10362 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10363 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10364 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10365 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10366 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10367 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10368 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10369 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10370 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10371 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10372 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10373 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10374 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10375 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10376 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10377 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10378 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10379 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10380 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10381 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10382 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10383 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10384 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10385 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10386 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10387 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10388 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10389 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10390 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10391 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10392 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10393 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10394 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10395 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10396 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10397 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10398 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10399 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10400 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10401 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10402 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10403 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10404 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10405 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10406 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10407 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10408 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10409 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10410 ) != 0;
10411}
10412
10413
10414/**
10415 * Runs the guest code using VT-x.
10416 *
10417 * @returns Strict VBox status code (i.e. informational status codes too).
10418 * @param pVM The cross context VM structure.
10419 * @param pVCpu The cross context virtual CPU structure.
10420 * @param pCtx Pointer to the guest-CPU context.
10421 */
10422VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10423{
10424 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10425 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10426 HMVMX_ASSERT_PREEMPT_SAFE();
10427
10428 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10429
10430 VBOXSTRICTRC rcStrict;
10431 if ( !pVCpu->hm.s.fUseDebugLoop
10432 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10433 && !DBGFIsStepping(pVCpu)
10434 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10435 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10436 else
10437 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10438
10439 if (rcStrict == VERR_EM_INTERPRETER)
10440 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10441 else if (rcStrict == VINF_EM_RESET)
10442 rcStrict = VINF_EM_TRIPLE_FAULT;
10443
10444 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10445 if (RT_FAILURE(rc2))
10446 {
10447 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10448 rcStrict = rc2;
10449 }
10450 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10451 return rcStrict;
10452}
10453
10454
10455#ifndef HMVMX_USE_FUNCTION_TABLE
10456DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10457{
10458# ifdef DEBUG_ramshankar
10459# define RETURN_EXIT_CALL(a_CallExpr) \
10460 do { \
10461 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10462 VBOXSTRICTRC rcStrict = a_CallExpr; \
10463 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10464 return rcStrict; \
10465 } while (0)
10466# else
10467# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10468# endif
10469 switch (rcReason)
10470 {
10471 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10472 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10473 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10474 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10475 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10476 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10477 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10478 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10479 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10480 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10481 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10482 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10483 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10484 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10485 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10486 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10487 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10488 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10489 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10490 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10491 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10492 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10493 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10494 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10495 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10496 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10497 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10498 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10499 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10500 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10501 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10502 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10503 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10504 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10505
10506 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10507 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10508 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10509 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10510 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10511 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10512 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10513 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10514 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10515
10516 case VMX_EXIT_VMCLEAR:
10517 case VMX_EXIT_VMLAUNCH:
10518 case VMX_EXIT_VMPTRLD:
10519 case VMX_EXIT_VMPTRST:
10520 case VMX_EXIT_VMREAD:
10521 case VMX_EXIT_VMRESUME:
10522 case VMX_EXIT_VMWRITE:
10523 case VMX_EXIT_VMXOFF:
10524 case VMX_EXIT_VMXON:
10525 case VMX_EXIT_INVEPT:
10526 case VMX_EXIT_INVVPID:
10527 case VMX_EXIT_VMFUNC:
10528 case VMX_EXIT_XSAVES:
10529 case VMX_EXIT_XRSTORS:
10530 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10531 case VMX_EXIT_ENCLS:
10532 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10533 case VMX_EXIT_PML_FULL:
10534 default:
10535 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10536 }
10537#undef RETURN_EXIT_CALL
10538}
10539#endif /* !HMVMX_USE_FUNCTION_TABLE */
10540
10541
10542#ifdef VBOX_STRICT
10543/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10544# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10545 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10546
10547# define HMVMX_ASSERT_PREEMPT_CPUID() \
10548 do { \
10549 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10550 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10551 } while (0)
10552
10553# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10554 do { \
10555 AssertPtr(pVCpu); \
10556 AssertPtr(pMixedCtx); \
10557 AssertPtr(pVmxTransient); \
10558 Assert(pVmxTransient->fVMEntryFailed == false); \
10559 Assert(ASMIntAreEnabled()); \
10560 HMVMX_ASSERT_PREEMPT_SAFE(); \
10561 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10562 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10563 HMVMX_ASSERT_PREEMPT_SAFE(); \
10564 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10565 HMVMX_ASSERT_PREEMPT_CPUID(); \
10566 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10567 } while (0)
10568
10569# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10570 do { \
10571 Log4Func(("\n")); \
10572 } while (0)
10573#else /* nonstrict builds: */
10574# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10575 do { \
10576 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10577 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10578 } while (0)
10579# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10580#endif
10581
10582
10583/**
10584 * Advances the guest RIP by the specified number of bytes.
10585 *
10586 * @param pVCpu The cross context virtual CPU structure.
10587 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10588 * out-of-sync. Make sure to update the required fields
10589 * before using them.
10590 * @param cbInstr Number of bytes to advance the RIP by.
10591 *
10592 * @remarks No-long-jump zone!!!
10593 */
10594DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10595{
10596 /* Advance the RIP. */
10597 pMixedCtx->rip += cbInstr;
10598 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10599
10600 /* Update interrupt inhibition. */
10601 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10602 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10603 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10604}
10605
10606
10607/**
10608 * Advances the guest RIP after reading it from the VMCS.
10609 *
10610 * @returns VBox status code, no informational status codes.
10611 * @param pVCpu The cross context virtual CPU structure.
10612 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10613 * out-of-sync. Make sure to update the required fields
10614 * before using them.
10615 * @param pVmxTransient Pointer to the VMX transient structure.
10616 *
10617 * @remarks No-long-jump zone!!!
10618 */
10619static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10620{
10621 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10622 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10623 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10624 AssertRCReturn(rc, rc);
10625
10626 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10627
10628 /*
10629 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10630 * pending debug exception field as it takes care of priority of events.
10631 *
10632 * See Intel spec. 32.2.1 "Debug Exceptions".
10633 */
10634 if ( !pVCpu->hm.s.fSingleInstruction
10635 && pMixedCtx->eflags.Bits.u1TF)
10636 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10637
10638 return VINF_SUCCESS;
10639}
10640
10641
10642/**
10643 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10644 * and update error record fields accordingly.
10645 *
10646 * @return VMX_IGS_* return codes.
10647 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10648 * wrong with the guest state.
10649 *
10650 * @param pVM The cross context VM structure.
10651 * @param pVCpu The cross context virtual CPU structure.
10652 * @param pCtx Pointer to the guest-CPU state.
10653 *
10654 * @remarks This function assumes our cache of the VMCS controls
10655 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10656 */
10657static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10658{
10659#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10660#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10661 uError = (err); \
10662 break; \
10663 } else do { } while (0)
10664
10665 int rc;
10666 uint32_t uError = VMX_IGS_ERROR;
10667 uint32_t u32Val;
10668 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10669
10670 do
10671 {
10672 /*
10673 * CR0.
10674 */
10675 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10676 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10677 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10678 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10679 if (fUnrestrictedGuest)
10680 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10681
10682 uint32_t u32GuestCR0;
10683 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10684 AssertRCBreak(rc);
10685 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10686 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10687 if ( !fUnrestrictedGuest
10688 && (u32GuestCR0 & X86_CR0_PG)
10689 && !(u32GuestCR0 & X86_CR0_PE))
10690 {
10691 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10692 }
10693
10694 /*
10695 * CR4.
10696 */
10697 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10698 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10699
10700 uint32_t u32GuestCR4;
10701 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10702 AssertRCBreak(rc);
10703 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10704 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10705
10706 /*
10707 * IA32_DEBUGCTL MSR.
10708 */
10709 uint64_t u64Val;
10710 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10711 AssertRCBreak(rc);
10712 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10713 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10714 {
10715 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10716 }
10717 uint64_t u64DebugCtlMsr = u64Val;
10718
10719#ifdef VBOX_STRICT
10720 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10721 AssertRCBreak(rc);
10722 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10723#endif
10724 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10725
10726 /*
10727 * RIP and RFLAGS.
10728 */
10729 uint32_t u32Eflags;
10730#if HC_ARCH_BITS == 64
10731 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10732 AssertRCBreak(rc);
10733 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10734 if ( !fLongModeGuest
10735 || !pCtx->cs.Attr.n.u1Long)
10736 {
10737 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10738 }
10739 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10740 * must be identical if the "IA-32e mode guest" VM-entry
10741 * control is 1 and CS.L is 1. No check applies if the
10742 * CPU supports 64 linear-address bits. */
10743
10744 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10745 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10746 AssertRCBreak(rc);
10747 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10748 VMX_IGS_RFLAGS_RESERVED);
10749 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10750 u32Eflags = u64Val;
10751#else
10752 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10753 AssertRCBreak(rc);
10754 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10755 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10756#endif
10757
10758 if ( fLongModeGuest
10759 || ( fUnrestrictedGuest
10760 && !(u32GuestCR0 & X86_CR0_PE)))
10761 {
10762 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10763 }
10764
10765 uint32_t u32EntryInfo;
10766 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10767 AssertRCBreak(rc);
10768 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10769 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10770 {
10771 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10772 }
10773
10774 /*
10775 * 64-bit checks.
10776 */
10777#if HC_ARCH_BITS == 64
10778 if (fLongModeGuest)
10779 {
10780 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10781 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10782 }
10783
10784 if ( !fLongModeGuest
10785 && (u32GuestCR4 & X86_CR4_PCIDE))
10786 {
10787 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10788 }
10789
10790 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10791 * 51:32 beyond the processor's physical-address width are 0. */
10792
10793 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10794 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10795 {
10796 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10797 }
10798
10799 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10800 AssertRCBreak(rc);
10801 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10802
10803 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10804 AssertRCBreak(rc);
10805 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10806#endif
10807
10808 /*
10809 * PERF_GLOBAL MSR.
10810 */
10811 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10812 {
10813 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10814 AssertRCBreak(rc);
10815 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10816 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10817 }
10818
10819 /*
10820 * PAT MSR.
10821 */
10822 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10823 {
10824 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10825 AssertRCBreak(rc);
10826 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10827 for (unsigned i = 0; i < 8; i++)
10828 {
10829 uint8_t u8Val = (u64Val & 0xff);
10830 if ( u8Val != 0 /* UC */
10831 && u8Val != 1 /* WC */
10832 && u8Val != 4 /* WT */
10833 && u8Val != 5 /* WP */
10834 && u8Val != 6 /* WB */
10835 && u8Val != 7 /* UC- */)
10836 {
10837 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10838 }
10839 u64Val >>= 8;
10840 }
10841 }
10842
10843 /*
10844 * EFER MSR.
10845 */
10846 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10847 {
10848 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10849 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10850 AssertRCBreak(rc);
10851 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10852 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10853 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10854 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10855 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10856 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10857 || !(u32GuestCR0 & X86_CR0_PG)
10858 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10859 VMX_IGS_EFER_LMA_LME_MISMATCH);
10860 }
10861
10862 /*
10863 * Segment registers.
10864 */
10865 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10866 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10867 if (!(u32Eflags & X86_EFL_VM))
10868 {
10869 /* CS */
10870 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10871 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10872 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10873 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10874 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10875 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10876 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10877 /* CS cannot be loaded with NULL in protected mode. */
10878 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10879 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10880 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10881 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10882 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10883 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10884 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10885 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10886 else
10887 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10888
10889 /* SS */
10890 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10891 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10892 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10893 if ( !(pCtx->cr0 & X86_CR0_PE)
10894 || pCtx->cs.Attr.n.u4Type == 3)
10895 {
10896 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10897 }
10898 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10899 {
10900 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10901 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10902 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10903 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10904 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10905 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10906 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10907 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10908 }
10909
10910 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10911 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10912 {
10913 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10914 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10915 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10916 || pCtx->ds.Attr.n.u4Type > 11
10917 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10918 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10919 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10920 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10921 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10922 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10923 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10924 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10925 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10926 }
10927 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10928 {
10929 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10930 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10931 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10932 || pCtx->es.Attr.n.u4Type > 11
10933 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10934 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10935 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10936 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10937 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10938 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10939 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10940 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10941 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10942 }
10943 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10944 {
10945 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10946 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10947 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10948 || pCtx->fs.Attr.n.u4Type > 11
10949 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10950 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10951 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10952 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10953 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10954 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10955 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10956 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10957 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10958 }
10959 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10960 {
10961 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10962 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10963 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10964 || pCtx->gs.Attr.n.u4Type > 11
10965 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10966 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10967 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10968 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10969 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10970 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10971 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10972 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10973 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10974 }
10975 /* 64-bit capable CPUs. */
10976#if HC_ARCH_BITS == 64
10977 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10978 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10979 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10980 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10981 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10982 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10983 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10984 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10985 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10986 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10987 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10988#endif
10989 }
10990 else
10991 {
10992 /* V86 mode checks. */
10993 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10994 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10995 {
10996 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10997 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10998 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10999 }
11000 else
11001 {
11002 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11003 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11004 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11005 }
11006
11007 /* CS */
11008 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11009 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11010 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11011 /* SS */
11012 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11013 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11014 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11015 /* DS */
11016 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11017 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11018 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11019 /* ES */
11020 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11021 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11022 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11023 /* FS */
11024 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11025 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11026 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11027 /* GS */
11028 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11029 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11030 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11031 /* 64-bit capable CPUs. */
11032#if HC_ARCH_BITS == 64
11033 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11034 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11035 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11036 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11037 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11038 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11039 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11040 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11041 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11042 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11043 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11044#endif
11045 }
11046
11047 /*
11048 * TR.
11049 */
11050 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11051 /* 64-bit capable CPUs. */
11052#if HC_ARCH_BITS == 64
11053 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11054#endif
11055 if (fLongModeGuest)
11056 {
11057 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11058 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11059 }
11060 else
11061 {
11062 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11063 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11064 VMX_IGS_TR_ATTR_TYPE_INVALID);
11065 }
11066 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11067 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11068 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11069 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11070 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11071 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11072 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11073 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11074
11075 /*
11076 * GDTR and IDTR.
11077 */
11078#if HC_ARCH_BITS == 64
11079 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11080 AssertRCBreak(rc);
11081 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11082
11083 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11084 AssertRCBreak(rc);
11085 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11086#endif
11087
11088 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11089 AssertRCBreak(rc);
11090 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11091
11092 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11093 AssertRCBreak(rc);
11094 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11095
11096 /*
11097 * Guest Non-Register State.
11098 */
11099 /* Activity State. */
11100 uint32_t u32ActivityState;
11101 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11102 AssertRCBreak(rc);
11103 HMVMX_CHECK_BREAK( !u32ActivityState
11104 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11105 VMX_IGS_ACTIVITY_STATE_INVALID);
11106 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11107 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11108 uint32_t u32IntrState;
11109 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11110 AssertRCBreak(rc);
11111 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11112 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11113 {
11114 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11115 }
11116
11117 /** @todo Activity state and injecting interrupts. Left as a todo since we
11118 * currently don't use activity states but ACTIVE. */
11119
11120 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11121 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11122
11123 /* Guest interruptibility-state. */
11124 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11125 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11126 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11127 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11128 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11129 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11130 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11131 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11132 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11133 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11134 {
11135 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11136 {
11137 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11138 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11139 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11140 }
11141 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11142 {
11143 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11144 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11145 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11146 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11147 }
11148 }
11149 /** @todo Assumes the processor is not in SMM. */
11150 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11151 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11152 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11153 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11154 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11155 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11156 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11157 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11158 {
11159 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11160 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11161 }
11162
11163 /* Pending debug exceptions. */
11164#if HC_ARCH_BITS == 64
11165 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11166 AssertRCBreak(rc);
11167 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11168 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11169 u32Val = u64Val; /* For pending debug exceptions checks below. */
11170#else
11171 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11172 AssertRCBreak(rc);
11173 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11174 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11175#endif
11176
11177 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11178 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11179 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11180 {
11181 if ( (u32Eflags & X86_EFL_TF)
11182 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11183 {
11184 /* Bit 14 is PendingDebug.BS. */
11185 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11186 }
11187 if ( !(u32Eflags & X86_EFL_TF)
11188 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11189 {
11190 /* Bit 14 is PendingDebug.BS. */
11191 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11192 }
11193 }
11194
11195 /* VMCS link pointer. */
11196 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11197 AssertRCBreak(rc);
11198 if (u64Val != UINT64_C(0xffffffffffffffff))
11199 {
11200 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11201 /** @todo Bits beyond the processor's physical-address width MBZ. */
11202 /** @todo 32-bit located in memory referenced by value of this field (as a
11203 * physical address) must contain the processor's VMCS revision ID. */
11204 /** @todo SMM checks. */
11205 }
11206
11207 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11208 * not using Nested Paging? */
11209 if ( pVM->hm.s.fNestedPaging
11210 && !fLongModeGuest
11211 && CPUMIsGuestInPAEModeEx(pCtx))
11212 {
11213 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11214 AssertRCBreak(rc);
11215 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11216
11217 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11218 AssertRCBreak(rc);
11219 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11220
11221 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11222 AssertRCBreak(rc);
11223 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11224
11225 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11226 AssertRCBreak(rc);
11227 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11228 }
11229
11230 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11231 if (uError == VMX_IGS_ERROR)
11232 uError = VMX_IGS_REASON_NOT_FOUND;
11233 } while (0);
11234
11235 pVCpu->hm.s.u32HMError = uError;
11236 return uError;
11237
11238#undef HMVMX_ERROR_BREAK
11239#undef HMVMX_CHECK_BREAK
11240}
11241
11242/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11243/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11244/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11245
11246/** @name VM-exit handlers.
11247 * @{
11248 */
11249
11250/**
11251 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11252 */
11253HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11254{
11255 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11257 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11258 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11259 return VINF_SUCCESS;
11260 return VINF_EM_RAW_INTERRUPT;
11261}
11262
11263
11264/**
11265 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11266 */
11267HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11268{
11269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11270 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11271
11272 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11273 AssertRCReturn(rc, rc);
11274
11275 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11276 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11277 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11278 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11279
11280 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11281 {
11282 /*
11283 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11284 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11285 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11286 *
11287 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11288 */
11289 VMXDispatchHostNmi();
11290 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11291 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11292 return VINF_SUCCESS;
11293 }
11294
11295 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11296 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11297 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11298 { /* likely */ }
11299 else
11300 {
11301 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11302 rcStrictRc1 = VINF_SUCCESS;
11303 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11304 return rcStrictRc1;
11305 }
11306
11307 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11308 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11309 switch (uIntType)
11310 {
11311 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11312 Assert(uVector == X86_XCPT_DB);
11313 /* no break */
11314 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11315 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11316 /* no break */
11317 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11318 {
11319 /*
11320 * If there's any exception caused as a result of event injection, go back to
11321 * the interpreter. The page-fault case is complicated and we manually handle
11322 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11323 * handled in hmR0VmxCheckExitDueToEventDelivery.
11324 */
11325 if (!pVCpu->hm.s.Event.fPending)
11326 { /* likely */ }
11327 else if ( uVector != X86_XCPT_PF
11328 && uVector != X86_XCPT_AC)
11329 {
11330 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11331 rc = VERR_EM_INTERPRETER;
11332 break;
11333 }
11334
11335 switch (uVector)
11336 {
11337 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11338 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11339 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11340 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11341 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11342 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11343 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11344
11345 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11346 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11347 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11348 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11349 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11350 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11351 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11352 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11353 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11354 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11355 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11356 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11357 default:
11358 {
11359 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11360 AssertRCReturn(rc, rc);
11361
11362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11363 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11364 {
11365 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11366 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11367 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11368
11369 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11370 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11371 AssertRCReturn(rc, rc);
11372 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11373 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11374 0 /* GCPtrFaultAddress */);
11375 AssertRCReturn(rc, rc);
11376 }
11377 else
11378 {
11379 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11380 pVCpu->hm.s.u32HMError = uVector;
11381 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11382 }
11383 break;
11384 }
11385 }
11386 break;
11387 }
11388
11389 default:
11390 {
11391 pVCpu->hm.s.u32HMError = uExitIntInfo;
11392 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11393 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11394 break;
11395 }
11396 }
11397 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11398 return rc;
11399}
11400
11401
11402/**
11403 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11404 */
11405HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11406{
11407 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11408
11409 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11410 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11411
11412 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11413 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11414 return VINF_SUCCESS;
11415}
11416
11417
11418/**
11419 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11420 */
11421HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11422{
11423 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11424 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11425 {
11426 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11427 HMVMX_RETURN_UNEXPECTED_EXIT();
11428 }
11429
11430 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11431
11432 /*
11433 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11434 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11435 */
11436 uint32_t uIntrState = 0;
11437 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11438 AssertRCReturn(rc, rc);
11439
11440 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11441 if ( fBlockSti
11442 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11443 {
11444 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11445 }
11446
11447 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11448 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11449
11450 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11451 return VINF_SUCCESS;
11452}
11453
11454
11455/**
11456 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11457 */
11458HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11459{
11460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11462 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11463}
11464
11465
11466/**
11467 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11468 */
11469HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11470{
11471 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11472 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11473 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11474}
11475
11476
11477/**
11478 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11479 */
11480HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11481{
11482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11483 PVM pVM = pVCpu->CTX_SUFF(pVM);
11484 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11485 if (RT_LIKELY(rc == VINF_SUCCESS))
11486 {
11487 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11488 Assert(pVmxTransient->cbInstr == 2);
11489 }
11490 else
11491 {
11492 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11493 rc = VERR_EM_INTERPRETER;
11494 }
11495 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11496 return rc;
11497}
11498
11499
11500/**
11501 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11502 */
11503HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11506 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11507 AssertRCReturn(rc, rc);
11508
11509 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11510 return VINF_EM_RAW_EMULATE_INSTR;
11511
11512 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11513 HMVMX_RETURN_UNEXPECTED_EXIT();
11514}
11515
11516
11517/**
11518 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11523 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11524 AssertRCReturn(rc, rc);
11525
11526 PVM pVM = pVCpu->CTX_SUFF(pVM);
11527 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11528 if (RT_LIKELY(rc == VINF_SUCCESS))
11529 {
11530 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11531 Assert(pVmxTransient->cbInstr == 2);
11532 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11533 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11534 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11535 }
11536 else
11537 rc = VERR_EM_INTERPRETER;
11538 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11539 return rc;
11540}
11541
11542
11543/**
11544 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11545 */
11546HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11547{
11548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11549 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11550 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11551 AssertRCReturn(rc, rc);
11552
11553 PVM pVM = pVCpu->CTX_SUFF(pVM);
11554 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11555 if (RT_SUCCESS(rc))
11556 {
11557 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11558 Assert(pVmxTransient->cbInstr == 3);
11559 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11560 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11561 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11562 }
11563 else
11564 {
11565 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11566 rc = VERR_EM_INTERPRETER;
11567 }
11568 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11569 return rc;
11570}
11571
11572
11573/**
11574 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11575 */
11576HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11577{
11578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11579 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11580 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11581 AssertRCReturn(rc, rc);
11582
11583 PVM pVM = pVCpu->CTX_SUFF(pVM);
11584 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11585 if (RT_LIKELY(rc == VINF_SUCCESS))
11586 {
11587 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11588 Assert(pVmxTransient->cbInstr == 2);
11589 }
11590 else
11591 {
11592 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11593 rc = VERR_EM_INTERPRETER;
11594 }
11595 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11596 return rc;
11597}
11598
11599
11600/**
11601 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11602 */
11603HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11604{
11605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11606 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11607
11608 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11609 if (pVCpu->hm.s.fHypercallsEnabled)
11610 {
11611#if 0
11612 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11613#else
11614 /* Aggressive state sync. for now. */
11615 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11616 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11617 AssertRCReturn(rc, rc);
11618#endif
11619
11620 /* Perform the hypercall. */
11621 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11622 if (rcStrict == VINF_SUCCESS)
11623 {
11624 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11625 AssertRCReturn(rc, rc);
11626 }
11627 else
11628 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11629 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11630 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11631
11632 /* If the hypercall changes anything other than guest's general-purpose registers,
11633 we would need to reload the guest changed bits here before VM-entry. */
11634 }
11635 else
11636 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11637
11638 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11639 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11640 {
11641 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11642 rcStrict = VINF_SUCCESS;
11643 }
11644
11645 return rcStrict;
11646}
11647
11648
11649/**
11650 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11651 */
11652HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11653{
11654 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11655 PVM pVM = pVCpu->CTX_SUFF(pVM);
11656 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11657
11658 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11659 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11660 AssertRCReturn(rc, rc);
11661
11662 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11663 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11664 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11665 else
11666 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11667 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11668 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11669 return rcStrict;
11670}
11671
11672
11673/**
11674 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11675 */
11676HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11677{
11678 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11679 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11680 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11681 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11682 AssertRCReturn(rc, rc);
11683
11684 PVM pVM = pVCpu->CTX_SUFF(pVM);
11685 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11686 if (RT_LIKELY(rc == VINF_SUCCESS))
11687 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11688 else
11689 {
11690 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11691 rc = VERR_EM_INTERPRETER;
11692 }
11693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11694 return rc;
11695}
11696
11697
11698/**
11699 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11700 */
11701HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11702{
11703 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11704 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11705 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11706 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11707 AssertRCReturn(rc, rc);
11708
11709 PVM pVM = pVCpu->CTX_SUFF(pVM);
11710 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11711 rc = VBOXSTRICTRC_VAL(rc2);
11712 if (RT_LIKELY( rc == VINF_SUCCESS
11713 || rc == VINF_EM_HALT))
11714 {
11715 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11716 AssertRCReturn(rc3, rc3);
11717
11718 if ( rc == VINF_EM_HALT
11719 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11720 {
11721 rc = VINF_SUCCESS;
11722 }
11723 }
11724 else
11725 {
11726 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11727 rc = VERR_EM_INTERPRETER;
11728 }
11729 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11730 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11731 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11732 return rc;
11733}
11734
11735
11736/**
11737 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11738 */
11739HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11740{
11741 /*
11742 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11743 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11744 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11745 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11746 */
11747 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11748 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11749 HMVMX_RETURN_UNEXPECTED_EXIT();
11750}
11751
11752
11753/**
11754 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11755 */
11756HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11757{
11758 /*
11759 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11760 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11761 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11762 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11763 */
11764 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11765 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11766 HMVMX_RETURN_UNEXPECTED_EXIT();
11767}
11768
11769
11770/**
11771 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11772 */
11773HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11774{
11775 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11776 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11777 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11778 HMVMX_RETURN_UNEXPECTED_EXIT();
11779}
11780
11781
11782/**
11783 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11784 */
11785HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11786{
11787 /*
11788 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11789 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11790 * See Intel spec. 25.3 "Other Causes of VM-exits".
11791 */
11792 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11793 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11794 HMVMX_RETURN_UNEXPECTED_EXIT();
11795}
11796
11797
11798/**
11799 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11800 * VM-exit.
11801 */
11802HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11803{
11804 /*
11805 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11806 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11807 *
11808 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11809 * See Intel spec. "23.8 Restrictions on VMX operation".
11810 */
11811 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11812 return VINF_SUCCESS;
11813}
11814
11815
11816/**
11817 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11818 * VM-exit.
11819 */
11820HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11821{
11822 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11823 return VINF_EM_RESET;
11824}
11825
11826
11827/**
11828 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11829 */
11830HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11831{
11832 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11833 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11834
11835 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11836 AssertRCReturn(rc, rc);
11837
11838 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11839 rc = VINF_SUCCESS;
11840 else
11841 rc = VINF_EM_HALT;
11842
11843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11844 if (rc != VINF_SUCCESS)
11845 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11846 return rc;
11847}
11848
11849
11850/**
11851 * VM-exit handler for instructions that result in a \#UD exception delivered to
11852 * the guest.
11853 */
11854HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11855{
11856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11858 return VINF_SUCCESS;
11859}
11860
11861
11862/**
11863 * VM-exit handler for expiry of the VMX preemption timer.
11864 */
11865HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11866{
11867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11868
11869 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11870 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11871
11872 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11873 PVM pVM = pVCpu->CTX_SUFF(pVM);
11874 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11875 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11876 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11877}
11878
11879
11880/**
11881 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11882 */
11883HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11884{
11885 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11886
11887 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11888 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11889 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11890 AssertRCReturn(rc, rc);
11891
11892 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11893 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11894
11895 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11896
11897 return rcStrict;
11898}
11899
11900
11901/**
11902 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11903 */
11904HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11905{
11906 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11907
11908 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11909 /** @todo implement EMInterpretInvpcid() */
11910 return VERR_EM_INTERPRETER;
11911}
11912
11913
11914/**
11915 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11916 * Error VM-exit.
11917 */
11918HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11919{
11920 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11921 AssertRCReturn(rc, rc);
11922
11923 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11924 AssertRCReturn(rc, rc);
11925
11926 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11927 NOREF(uInvalidReason);
11928
11929#ifdef VBOX_STRICT
11930 uint32_t uIntrState;
11931 RTHCUINTREG uHCReg;
11932 uint64_t u64Val;
11933 uint32_t u32Val;
11934
11935 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11936 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11937 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11938 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11939 AssertRCReturn(rc, rc);
11940
11941 Log4(("uInvalidReason %u\n", uInvalidReason));
11942 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11943 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11944 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11945 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11946
11947 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11948 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11949 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11950 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11951 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11952 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11953 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11954 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11955 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11956 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11957 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11958 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11959#else
11960 NOREF(pVmxTransient);
11961#endif
11962
11963 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11964 return VERR_VMX_INVALID_GUEST_STATE;
11965}
11966
11967
11968/**
11969 * VM-exit handler for VM-entry failure due to an MSR-load
11970 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11971 */
11972HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11973{
11974 NOREF(pVmxTransient);
11975 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11976 HMVMX_RETURN_UNEXPECTED_EXIT();
11977}
11978
11979
11980/**
11981 * VM-exit handler for VM-entry failure due to a machine-check event
11982 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11983 */
11984HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11985{
11986 NOREF(pVmxTransient);
11987 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11988 HMVMX_RETURN_UNEXPECTED_EXIT();
11989}
11990
11991
11992/**
11993 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11994 * theory.
11995 */
11996HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11997{
11998 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11999 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12000 return VERR_VMX_UNDEFINED_EXIT_CODE;
12001}
12002
12003
12004/**
12005 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12006 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12007 * Conditional VM-exit.
12008 */
12009HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12010{
12011 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12012
12013 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12015 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12016 return VERR_EM_INTERPRETER;
12017 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12018 HMVMX_RETURN_UNEXPECTED_EXIT();
12019}
12020
12021
12022/**
12023 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12024 */
12025HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12026{
12027 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12028
12029 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12031 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12032 return VERR_EM_INTERPRETER;
12033 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12034 HMVMX_RETURN_UNEXPECTED_EXIT();
12035}
12036
12037
12038/**
12039 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12040 */
12041HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12042{
12043 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12044
12045 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12046 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12047 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12048 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12049 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12050 {
12051 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12052 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12053 }
12054 AssertRCReturn(rc, rc);
12055 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12056
12057#ifdef VBOX_STRICT
12058 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12059 {
12060 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12061 && pMixedCtx->ecx != MSR_K6_EFER)
12062 {
12063 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12064 pMixedCtx->ecx));
12065 HMVMX_RETURN_UNEXPECTED_EXIT();
12066 }
12067 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12068 {
12069 VMXMSREXITREAD enmRead;
12070 VMXMSREXITWRITE enmWrite;
12071 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12072 AssertRCReturn(rc2, rc2);
12073 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12074 {
12075 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12076 HMVMX_RETURN_UNEXPECTED_EXIT();
12077 }
12078 }
12079 }
12080#endif
12081
12082 PVM pVM = pVCpu->CTX_SUFF(pVM);
12083 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12084 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12085 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12087 if (RT_SUCCESS(rc))
12088 {
12089 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12090 Assert(pVmxTransient->cbInstr == 2);
12091 }
12092 return rc;
12093}
12094
12095
12096/**
12097 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12098 */
12099HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12100{
12101 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12102 PVM pVM = pVCpu->CTX_SUFF(pVM);
12103 int rc = VINF_SUCCESS;
12104
12105 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12106 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12107 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12108 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12109 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12110 {
12111 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12112 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12113 }
12114 AssertRCReturn(rc, rc);
12115 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12116
12117 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12118 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12120
12121 if (RT_SUCCESS(rc))
12122 {
12123 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12124
12125 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12126 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12127 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12128 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12129 {
12130 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12131 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12132 EMInterpretWrmsr() changes it. */
12133 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12134 }
12135 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12136 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12137 else if (pMixedCtx->ecx == MSR_K6_EFER)
12138 {
12139 /*
12140 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12141 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12142 * the other bits as well, SCE and NXE. See @bugref{7368}.
12143 */
12144 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12145 }
12146
12147 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12148 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12149 {
12150 switch (pMixedCtx->ecx)
12151 {
12152 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12153 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12154 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12155 case MSR_K8_FS_BASE: /* no break */
12156 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12157 case MSR_K6_EFER: /* already handled above */ break;
12158 default:
12159 {
12160 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12161 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12162 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12163 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12164 break;
12165 }
12166 }
12167 }
12168#ifdef VBOX_STRICT
12169 else
12170 {
12171 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12172 switch (pMixedCtx->ecx)
12173 {
12174 case MSR_IA32_SYSENTER_CS:
12175 case MSR_IA32_SYSENTER_EIP:
12176 case MSR_IA32_SYSENTER_ESP:
12177 case MSR_K8_FS_BASE:
12178 case MSR_K8_GS_BASE:
12179 {
12180 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12181 HMVMX_RETURN_UNEXPECTED_EXIT();
12182 }
12183
12184 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12185 default:
12186 {
12187 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12188 {
12189 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12190 if (pMixedCtx->ecx != MSR_K6_EFER)
12191 {
12192 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12193 pMixedCtx->ecx));
12194 HMVMX_RETURN_UNEXPECTED_EXIT();
12195 }
12196 }
12197
12198 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12199 {
12200 VMXMSREXITREAD enmRead;
12201 VMXMSREXITWRITE enmWrite;
12202 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12203 AssertRCReturn(rc2, rc2);
12204 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12205 {
12206 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12207 HMVMX_RETURN_UNEXPECTED_EXIT();
12208 }
12209 }
12210 break;
12211 }
12212 }
12213 }
12214#endif /* VBOX_STRICT */
12215 }
12216 return rc;
12217}
12218
12219
12220/**
12221 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12222 */
12223HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12224{
12225 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12226
12227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12228 return VINF_EM_RAW_INTERRUPT;
12229}
12230
12231
12232/**
12233 * VM-exit handler for when the TPR value is lowered below the specified
12234 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12235 */
12236HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12237{
12238 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12239 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12240
12241 /*
12242 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12243 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12244 * resume guest execution.
12245 */
12246 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12248 return VINF_SUCCESS;
12249}
12250
12251
12252/**
12253 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12254 * VM-exit.
12255 *
12256 * @retval VINF_SUCCESS when guest execution can continue.
12257 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12258 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12259 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12260 * interpreter.
12261 */
12262HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12263{
12264 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12265 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12266 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12267 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12268 AssertRCReturn(rc, rc);
12269
12270 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12271 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12272 PVM pVM = pVCpu->CTX_SUFF(pVM);
12273 VBOXSTRICTRC rcStrict;
12274 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12275 switch (uAccessType)
12276 {
12277 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12278 {
12279 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12280 AssertRCReturn(rc, rc);
12281
12282 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12283 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12284 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12285 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12286 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12287 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12288 {
12289 case 0: /* CR0 */
12290 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12291 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12292 break;
12293 case 2: /* CR2 */
12294 /* Nothing to do here, CR2 it's not part of the VMCS. */
12295 break;
12296 case 3: /* CR3 */
12297 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12298 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12299 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12300 break;
12301 case 4: /* CR4 */
12302 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12303 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12304 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12305 break;
12306 case 8: /* CR8 */
12307 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12308 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12309 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12310 break;
12311 default:
12312 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12313 break;
12314 }
12315
12316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12317 break;
12318 }
12319
12320 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12321 {
12322 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12323 AssertRCReturn(rc, rc);
12324
12325 Assert( !pVM->hm.s.fNestedPaging
12326 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12327 || pVCpu->hm.s.fUsingDebugLoop
12328 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12329
12330 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12331 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12332 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12333
12334 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12335 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12336 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12337 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12339 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12340 VBOXSTRICTRC_VAL(rcStrict)));
12341 break;
12342 }
12343
12344 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12345 {
12346 AssertRCReturn(rc, rc);
12347 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12348 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12349 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12350 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12351 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12352 break;
12353 }
12354
12355 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12356 {
12357 AssertRCReturn(rc, rc);
12358 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12359 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12360 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12361 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12363 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12364 break;
12365 }
12366
12367 default:
12368 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12369 VERR_VMX_UNEXPECTED_EXCEPTION);
12370 }
12371
12372 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12373 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12374 NOREF(pVM);
12375 return rcStrict;
12376}
12377
12378
12379/**
12380 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12381 * VM-exit.
12382 */
12383HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12384{
12385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12386 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12387
12388 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12389 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12390 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12391 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12392 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12393 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12394 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12395 AssertRCReturn(rc2, rc2);
12396
12397 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12398 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12399 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12400 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12401 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12402 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12403 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12404 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12405 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12406
12407 /* I/O operation lookup arrays. */
12408 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12409 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12410
12411 VBOXSTRICTRC rcStrict;
12412 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12413 uint32_t const cbInstr = pVmxTransient->cbInstr;
12414 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12415 PVM pVM = pVCpu->CTX_SUFF(pVM);
12416 if (fIOString)
12417 {
12418#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12419 See @bugref{5752#c158}. Should work now. */
12420 /*
12421 * INS/OUTS - I/O String instruction.
12422 *
12423 * Use instruction-information if available, otherwise fall back on
12424 * interpreting the instruction.
12425 */
12426 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12427 fIOWrite ? 'w' : 'r'));
12428 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12429 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12430 {
12431 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12432 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12433 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12434 AssertRCReturn(rc2, rc2);
12435 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12436 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12437 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12438 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12439 if (fIOWrite)
12440 {
12441 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12442 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12443 }
12444 else
12445 {
12446 /*
12447 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12448 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12449 * See Intel Instruction spec. for "INS".
12450 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12451 */
12452 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12453 }
12454 }
12455 else
12456 {
12457 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12458 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12459 AssertRCReturn(rc2, rc2);
12460 rcStrict = IEMExecOne(pVCpu);
12461 }
12462 /** @todo IEM needs to be setting these flags somehow. */
12463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12464 fUpdateRipAlready = true;
12465#else
12466 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12467 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12468 if (RT_SUCCESS(rcStrict))
12469 {
12470 if (fIOWrite)
12471 {
12472 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12473 (DISCPUMODE)pDis->uAddrMode, cbValue);
12474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12475 }
12476 else
12477 {
12478 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12479 (DISCPUMODE)pDis->uAddrMode, cbValue);
12480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12481 }
12482 }
12483 else
12484 {
12485 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12486 pMixedCtx->rip));
12487 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12488 }
12489#endif
12490 }
12491 else
12492 {
12493 /*
12494 * IN/OUT - I/O instruction.
12495 */
12496 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12497 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12498 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12499 if (fIOWrite)
12500 {
12501 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12503 }
12504 else
12505 {
12506 uint32_t u32Result = 0;
12507 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12508 if (IOM_SUCCESS(rcStrict))
12509 {
12510 /* Save result of I/O IN instr. in AL/AX/EAX. */
12511 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12512 }
12513 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12514 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12515 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12516 }
12517 }
12518
12519 if (IOM_SUCCESS(rcStrict))
12520 {
12521 if (!fUpdateRipAlready)
12522 {
12523 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12524 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12525 }
12526
12527 /*
12528 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12529 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12530 */
12531 if (fIOString)
12532 {
12533 /** @todo Single-step for INS/OUTS with REP prefix? */
12534 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12535 }
12536 else if ( !fDbgStepping
12537 && fGstStepping)
12538 {
12539 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12540 }
12541
12542 /*
12543 * If any I/O breakpoints are armed, we need to check if one triggered
12544 * and take appropriate action.
12545 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12546 */
12547 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12548 AssertRCReturn(rc2, rc2);
12549
12550 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12551 * execution engines about whether hyper BPs and such are pending. */
12552 uint32_t const uDr7 = pMixedCtx->dr[7];
12553 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12554 && X86_DR7_ANY_RW_IO(uDr7)
12555 && (pMixedCtx->cr4 & X86_CR4_DE))
12556 || DBGFBpIsHwIoArmed(pVM)))
12557 {
12558 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12559
12560 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12561 VMMRZCallRing3Disable(pVCpu);
12562 HM_DISABLE_PREEMPT();
12563
12564 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12565
12566 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12567 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12568 {
12569 /* Raise #DB. */
12570 if (fIsGuestDbgActive)
12571 ASMSetDR6(pMixedCtx->dr[6]);
12572 if (pMixedCtx->dr[7] != uDr7)
12573 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12574
12575 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12576 }
12577 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12578 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12579 else if ( rcStrict2 != VINF_SUCCESS
12580 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12581 rcStrict = rcStrict2;
12582 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12583
12584 HM_RESTORE_PREEMPT();
12585 VMMRZCallRing3Enable(pVCpu);
12586 }
12587 }
12588
12589#ifdef VBOX_STRICT
12590 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12591 Assert(!fIOWrite);
12592 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12593 Assert(fIOWrite);
12594 else
12595 {
12596#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12597 * statuses, that the VMM device and some others may return. See
12598 * IOM_SUCCESS() for guidance. */
12599 AssertMsg( RT_FAILURE(rcStrict)
12600 || rcStrict == VINF_SUCCESS
12601 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12602 || rcStrict == VINF_EM_DBG_BREAKPOINT
12603 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12604 || rcStrict == VINF_EM_RAW_TO_R3
12605 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12606#endif
12607 }
12608#endif
12609
12610 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12611 return rcStrict;
12612}
12613
12614
12615/**
12616 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12617 * VM-exit.
12618 */
12619HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12620{
12621 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12622
12623 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12624 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12625 AssertRCReturn(rc, rc);
12626 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12627 {
12628 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12629 AssertRCReturn(rc, rc);
12630 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12631 {
12632 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12633
12634 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12635 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12636
12637 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12638 Assert(!pVCpu->hm.s.Event.fPending);
12639 pVCpu->hm.s.Event.fPending = true;
12640 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12641 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12642 AssertRCReturn(rc, rc);
12643 if (fErrorCodeValid)
12644 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12645 else
12646 pVCpu->hm.s.Event.u32ErrCode = 0;
12647 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12648 && uVector == X86_XCPT_PF)
12649 {
12650 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12651 }
12652
12653 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12654 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12655 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12656 }
12657 }
12658
12659 /* Fall back to the interpreter to emulate the task-switch. */
12660 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12661 return VERR_EM_INTERPRETER;
12662}
12663
12664
12665/**
12666 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12667 */
12668HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12669{
12670 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12671 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12672 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12673 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12674 AssertRCReturn(rc, rc);
12675 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12676 return VINF_EM_DBG_STEPPED;
12677}
12678
12679
12680/**
12681 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12682 */
12683HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12684{
12685 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12686
12687 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12688
12689 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12690 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12691 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12692 {
12693 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12694 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12695 {
12696 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12697 return VERR_EM_INTERPRETER;
12698 }
12699 }
12700 else
12701 {
12702 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12703 rcStrict1 = VINF_SUCCESS;
12704 return rcStrict1;
12705 }
12706
12707#if 0
12708 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12709 * just sync the whole thing. */
12710 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12711#else
12712 /* Aggressive state sync. for now. */
12713 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12714 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12715 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12716#endif
12717 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12718 AssertRCReturn(rc, rc);
12719
12720 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12721 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12722 VBOXSTRICTRC rcStrict2;
12723 switch (uAccessType)
12724 {
12725 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12726 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12727 {
12728 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12729 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12730 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12731
12732 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12733 GCPhys &= PAGE_BASE_GC_MASK;
12734 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12735 PVM pVM = pVCpu->CTX_SUFF(pVM);
12736 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12737 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12738
12739 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12740 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12741 CPUMCTX2CORE(pMixedCtx), GCPhys);
12742 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12743 if ( rcStrict2 == VINF_SUCCESS
12744 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12745 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12746 {
12747 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12748 | HM_CHANGED_GUEST_RSP
12749 | HM_CHANGED_GUEST_RFLAGS
12750 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12751 rcStrict2 = VINF_SUCCESS;
12752 }
12753 break;
12754 }
12755
12756 default:
12757 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12758 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12759 break;
12760 }
12761
12762 if (rcStrict2 != VINF_SUCCESS)
12763 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12764 return rcStrict2;
12765}
12766
12767
12768/**
12769 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12770 * VM-exit.
12771 */
12772HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12773{
12774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12775
12776 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12777 if (pVmxTransient->fWasGuestDebugStateActive)
12778 {
12779 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12780 HMVMX_RETURN_UNEXPECTED_EXIT();
12781 }
12782
12783 if ( !pVCpu->hm.s.fSingleInstruction
12784 && !pVmxTransient->fWasHyperDebugStateActive)
12785 {
12786 Assert(!DBGFIsStepping(pVCpu));
12787 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12788
12789 /* Don't intercept MOV DRx any more. */
12790 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12791 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12792 AssertRCReturn(rc, rc);
12793
12794 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12795 VMMRZCallRing3Disable(pVCpu);
12796 HM_DISABLE_PREEMPT();
12797
12798 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12799 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12800 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12801
12802 HM_RESTORE_PREEMPT();
12803 VMMRZCallRing3Enable(pVCpu);
12804
12805#ifdef VBOX_WITH_STATISTICS
12806 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12807 AssertRCReturn(rc, rc);
12808 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12809 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12810 else
12811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12812#endif
12813 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12814 return VINF_SUCCESS;
12815 }
12816
12817 /*
12818 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12819 * Update the segment registers and DR7 from the CPU.
12820 */
12821 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12822 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12823 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12824 AssertRCReturn(rc, rc);
12825 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12826
12827 PVM pVM = pVCpu->CTX_SUFF(pVM);
12828 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12829 {
12830 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12831 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12832 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12833 if (RT_SUCCESS(rc))
12834 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12835 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12836 }
12837 else
12838 {
12839 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12840 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12841 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12843 }
12844
12845 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12846 if (RT_SUCCESS(rc))
12847 {
12848 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12849 AssertRCReturn(rc2, rc2);
12850 return VINF_SUCCESS;
12851 }
12852 return rc;
12853}
12854
12855
12856/**
12857 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12858 * Conditional VM-exit.
12859 */
12860HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12861{
12862 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12863 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12864
12865 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12866 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12867 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12868 {
12869 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12870 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12871 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12872 {
12873 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12874 return VERR_EM_INTERPRETER;
12875 }
12876 }
12877 else
12878 {
12879 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12880 rcStrict1 = VINF_SUCCESS;
12881 return rcStrict1;
12882 }
12883
12884 RTGCPHYS GCPhys = 0;
12885 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12886
12887#if 0
12888 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12889#else
12890 /* Aggressive state sync. for now. */
12891 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12892 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12893 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12894#endif
12895 AssertRCReturn(rc, rc);
12896
12897 /*
12898 * If we succeed, resume guest execution.
12899 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12900 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12901 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12902 * weird case. See @bugref{6043}.
12903 */
12904 PVM pVM = pVCpu->CTX_SUFF(pVM);
12905 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12906 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12907 if ( rcStrict2 == VINF_SUCCESS
12908 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12909 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12910 {
12911 /* Successfully handled MMIO operation. */
12912 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12913 | HM_CHANGED_GUEST_RSP
12914 | HM_CHANGED_GUEST_RFLAGS
12915 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12916 return VINF_SUCCESS;
12917 }
12918 return rcStrict2;
12919}
12920
12921
12922/**
12923 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12924 * VM-exit.
12925 */
12926HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12927{
12928 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12929 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12930
12931 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12932 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12933 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12934 {
12935 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12936 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12937 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12938 }
12939 else
12940 {
12941 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12942 rcStrict1 = VINF_SUCCESS;
12943 return rcStrict1;
12944 }
12945
12946 RTGCPHYS GCPhys = 0;
12947 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12948 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12949#if 0
12950 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12951#else
12952 /* Aggressive state sync. for now. */
12953 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12954 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12955 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12956#endif
12957 AssertRCReturn(rc, rc);
12958
12959 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12960 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12961
12962 RTGCUINT uErrorCode = 0;
12963 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12964 uErrorCode |= X86_TRAP_PF_ID;
12965 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12966 uErrorCode |= X86_TRAP_PF_RW;
12967 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12968 uErrorCode |= X86_TRAP_PF_P;
12969
12970 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12971
12972 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12973 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12974
12975 /* Handle the pagefault trap for the nested shadow table. */
12976 PVM pVM = pVCpu->CTX_SUFF(pVM);
12977 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12978 TRPMResetTrap(pVCpu);
12979
12980 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12981 if ( rcStrict2 == VINF_SUCCESS
12982 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12983 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12984 {
12985 /* Successfully synced our nested page tables. */
12986 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12987 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12988 | HM_CHANGED_GUEST_RSP
12989 | HM_CHANGED_GUEST_RFLAGS);
12990 return VINF_SUCCESS;
12991 }
12992
12993 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12994 return rcStrict2;
12995}
12996
12997/** @} */
12998
12999/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13000/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13001/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13002
13003/** @name VM-exit exception handlers.
13004 * @{
13005 */
13006
13007/**
13008 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13009 */
13010static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13011{
13012 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13013 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13014
13015 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13016 AssertRCReturn(rc, rc);
13017
13018 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13019 {
13020 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13021 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13022
13023 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13024 * provides VM-exit instruction length. If this causes problem later,
13025 * disassemble the instruction like it's done on AMD-V. */
13026 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13027 AssertRCReturn(rc2, rc2);
13028 return rc;
13029 }
13030
13031 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13032 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13033 return rc;
13034}
13035
13036
13037/**
13038 * VM-exit exception handler for \#BP (Breakpoint exception).
13039 */
13040static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13041{
13042 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13044
13045 /** @todo Try optimize this by not saving the entire guest state unless
13046 * really needed. */
13047 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13048 AssertRCReturn(rc, rc);
13049
13050 PVM pVM = pVCpu->CTX_SUFF(pVM);
13051 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13052 if (rc == VINF_EM_RAW_GUEST_TRAP)
13053 {
13054 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13055 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13056 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13057 AssertRCReturn(rc, rc);
13058
13059 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13060 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13061 }
13062
13063 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13064 return rc;
13065}
13066
13067
13068/**
13069 * VM-exit exception handler for \#AC (alignment check exception).
13070 */
13071static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13072{
13073 RT_NOREF_PV(pMixedCtx);
13074 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13075
13076 /*
13077 * Re-inject it. We'll detect any nesting before getting here.
13078 */
13079 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13080 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13081 AssertRCReturn(rc, rc);
13082 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13083
13084 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13085 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13086 return VINF_SUCCESS;
13087}
13088
13089
13090/**
13091 * VM-exit exception handler for \#DB (Debug exception).
13092 */
13093static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13094{
13095 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13097 Log6(("XcptDB\n"));
13098
13099 /*
13100 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13101 * for processing.
13102 */
13103 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13104 AssertRCReturn(rc, rc);
13105
13106 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13107 uint64_t uDR6 = X86_DR6_INIT_VAL;
13108 uDR6 |= ( pVmxTransient->uExitQualification
13109 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13110
13111 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13112 if (rc == VINF_EM_RAW_GUEST_TRAP)
13113 {
13114 /*
13115 * The exception was for the guest. Update DR6, DR7.GD and
13116 * IA32_DEBUGCTL.LBR before forwarding it.
13117 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13118 */
13119 VMMRZCallRing3Disable(pVCpu);
13120 HM_DISABLE_PREEMPT();
13121
13122 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13123 pMixedCtx->dr[6] |= uDR6;
13124 if (CPUMIsGuestDebugStateActive(pVCpu))
13125 ASMSetDR6(pMixedCtx->dr[6]);
13126
13127 HM_RESTORE_PREEMPT();
13128 VMMRZCallRing3Enable(pVCpu);
13129
13130 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13131 AssertRCReturn(rc, rc);
13132
13133 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13134 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13135
13136 /* Paranoia. */
13137 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13138 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13139
13140 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13141 AssertRCReturn(rc, rc);
13142
13143 /*
13144 * Raise #DB in the guest.
13145 *
13146 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13147 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13148 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13149 *
13150 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13151 */
13152 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13153 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13154 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13155 AssertRCReturn(rc, rc);
13156 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13157 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13158 return VINF_SUCCESS;
13159 }
13160
13161 /*
13162 * Not a guest trap, must be a hypervisor related debug event then.
13163 * Update DR6 in case someone is interested in it.
13164 */
13165 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13166 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13167 CPUMSetHyperDR6(pVCpu, uDR6);
13168
13169 return rc;
13170}
13171
13172
13173/**
13174 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13175 * point exception).
13176 */
13177static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13178{
13179 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13180
13181 /* We require CR0 and EFER. EFER is always up-to-date. */
13182 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13183 AssertRCReturn(rc, rc);
13184
13185 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13186 VMMRZCallRing3Disable(pVCpu);
13187 HM_DISABLE_PREEMPT();
13188
13189 /* If the guest FPU was active at the time of the #NM VM-exit, then it's a guest fault. */
13190 if (pVmxTransient->fWasGuestFPUStateActive)
13191 {
13192 rc = VINF_EM_RAW_GUEST_TRAP;
13193 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13194 }
13195 else
13196 {
13197#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13198 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13199#endif
13200 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13201 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13202 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13203 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13204 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13205 }
13206
13207 HM_RESTORE_PREEMPT();
13208 VMMRZCallRing3Enable(pVCpu);
13209
13210 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13211 {
13212 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13213 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13215 pVCpu->hm.s.fPreloadGuestFpu = true;
13216 }
13217 else
13218 {
13219 /* Forward #NM to the guest. */
13220 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13221 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13222 AssertRCReturn(rc, rc);
13223 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13224 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13225 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13226 }
13227
13228 return VINF_SUCCESS;
13229}
13230
13231
13232/**
13233 * VM-exit exception handler for \#GP (General-protection exception).
13234 *
13235 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13236 */
13237static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13238{
13239 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13241
13242 int rc;
13243 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13244 { /* likely */ }
13245 else
13246 {
13247#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13248 Assert(pVCpu->hm.s.fUsingDebugLoop);
13249#endif
13250 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13251 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13252 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13253 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13254 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13255 AssertRCReturn(rc, rc);
13256 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13257 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13258 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13259 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13260 return rc;
13261 }
13262
13263 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13264 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13265
13266 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13267 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13268 AssertRCReturn(rc, rc);
13269
13270 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13271 uint32_t cbOp = 0;
13272 PVM pVM = pVCpu->CTX_SUFF(pVM);
13273 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13274 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13275 if (RT_SUCCESS(rc))
13276 {
13277 rc = VINF_SUCCESS;
13278 Assert(cbOp == pDis->cbInstr);
13279 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13280 switch (pDis->pCurInstr->uOpcode)
13281 {
13282 case OP_CLI:
13283 {
13284 pMixedCtx->eflags.Bits.u1IF = 0;
13285 pMixedCtx->eflags.Bits.u1RF = 0;
13286 pMixedCtx->rip += pDis->cbInstr;
13287 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13288 if ( !fDbgStepping
13289 && pMixedCtx->eflags.Bits.u1TF)
13290 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13291 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13292 break;
13293 }
13294
13295 case OP_STI:
13296 {
13297 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13298 pMixedCtx->eflags.Bits.u1IF = 1;
13299 pMixedCtx->eflags.Bits.u1RF = 0;
13300 pMixedCtx->rip += pDis->cbInstr;
13301 if (!fOldIF)
13302 {
13303 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13304 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13305 }
13306 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13307 if ( !fDbgStepping
13308 && pMixedCtx->eflags.Bits.u1TF)
13309 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13310 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13311 break;
13312 }
13313
13314 case OP_HLT:
13315 {
13316 rc = VINF_EM_HALT;
13317 pMixedCtx->rip += pDis->cbInstr;
13318 pMixedCtx->eflags.Bits.u1RF = 0;
13319 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13321 break;
13322 }
13323
13324 case OP_POPF:
13325 {
13326 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13327 uint32_t cbParm;
13328 uint32_t uMask;
13329 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13330 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13331 {
13332 cbParm = 4;
13333 uMask = 0xffffffff;
13334 }
13335 else
13336 {
13337 cbParm = 2;
13338 uMask = 0xffff;
13339 }
13340
13341 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13342 RTGCPTR GCPtrStack = 0;
13343 X86EFLAGS Eflags;
13344 Eflags.u32 = 0;
13345 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13346 &GCPtrStack);
13347 if (RT_SUCCESS(rc))
13348 {
13349 Assert(sizeof(Eflags.u32) >= cbParm);
13350 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13351 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13352 }
13353 if (RT_FAILURE(rc))
13354 {
13355 rc = VERR_EM_INTERPRETER;
13356 break;
13357 }
13358 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13359 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13360 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13361 pMixedCtx->esp += cbParm;
13362 pMixedCtx->esp &= uMask;
13363 pMixedCtx->rip += pDis->cbInstr;
13364 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13365 | HM_CHANGED_GUEST_RSP
13366 | HM_CHANGED_GUEST_RFLAGS);
13367 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13368 POPF restores EFLAGS.TF. */
13369 if ( !fDbgStepping
13370 && fGstStepping)
13371 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13372 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13373 break;
13374 }
13375
13376 case OP_PUSHF:
13377 {
13378 uint32_t cbParm;
13379 uint32_t uMask;
13380 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13381 {
13382 cbParm = 4;
13383 uMask = 0xffffffff;
13384 }
13385 else
13386 {
13387 cbParm = 2;
13388 uMask = 0xffff;
13389 }
13390
13391 /* Get the stack pointer & push the contents of eflags onto the stack. */
13392 RTGCPTR GCPtrStack = 0;
13393 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13394 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13395 if (RT_FAILURE(rc))
13396 {
13397 rc = VERR_EM_INTERPRETER;
13398 break;
13399 }
13400 X86EFLAGS Eflags = pMixedCtx->eflags;
13401 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13402 Eflags.Bits.u1RF = 0;
13403 Eflags.Bits.u1VM = 0;
13404
13405 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13406 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13407 {
13408 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13409 rc = VERR_EM_INTERPRETER;
13410 break;
13411 }
13412 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13413 pMixedCtx->esp -= cbParm;
13414 pMixedCtx->esp &= uMask;
13415 pMixedCtx->rip += pDis->cbInstr;
13416 pMixedCtx->eflags.Bits.u1RF = 0;
13417 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13418 | HM_CHANGED_GUEST_RSP
13419 | HM_CHANGED_GUEST_RFLAGS);
13420 if ( !fDbgStepping
13421 && pMixedCtx->eflags.Bits.u1TF)
13422 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13423 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13424 break;
13425 }
13426
13427 case OP_IRET:
13428 {
13429 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13430 * instruction reference. */
13431 RTGCPTR GCPtrStack = 0;
13432 uint32_t uMask = 0xffff;
13433 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13434 uint16_t aIretFrame[3];
13435 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13436 {
13437 rc = VERR_EM_INTERPRETER;
13438 break;
13439 }
13440 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13441 &GCPtrStack);
13442 if (RT_SUCCESS(rc))
13443 {
13444 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13445 PGMACCESSORIGIN_HM));
13446 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13447 }
13448 if (RT_FAILURE(rc))
13449 {
13450 rc = VERR_EM_INTERPRETER;
13451 break;
13452 }
13453 pMixedCtx->eip = 0;
13454 pMixedCtx->ip = aIretFrame[0];
13455 pMixedCtx->cs.Sel = aIretFrame[1];
13456 pMixedCtx->cs.ValidSel = aIretFrame[1];
13457 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13458 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13459 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13460 pMixedCtx->sp += sizeof(aIretFrame);
13461 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13462 | HM_CHANGED_GUEST_SEGMENT_REGS
13463 | HM_CHANGED_GUEST_RSP
13464 | HM_CHANGED_GUEST_RFLAGS);
13465 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13466 if ( !fDbgStepping
13467 && fGstStepping)
13468 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13469 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13470 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13471 break;
13472 }
13473
13474 case OP_INT:
13475 {
13476 uint16_t uVector = pDis->Param1.uValue & 0xff;
13477 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13478 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13480 break;
13481 }
13482
13483 case OP_INTO:
13484 {
13485 if (pMixedCtx->eflags.Bits.u1OF)
13486 {
13487 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13488 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13490 }
13491 else
13492 {
13493 pMixedCtx->eflags.Bits.u1RF = 0;
13494 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13495 }
13496 break;
13497 }
13498
13499 default:
13500 {
13501 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13502 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13503 EMCODETYPE_SUPERVISOR);
13504 rc = VBOXSTRICTRC_VAL(rc2);
13505 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13506 /** @todo We have to set pending-debug exceptions here when the guest is
13507 * single-stepping depending on the instruction that was interpreted. */
13508 Log4(("#GP rc=%Rrc\n", rc));
13509 break;
13510 }
13511 }
13512 }
13513 else
13514 rc = VERR_EM_INTERPRETER;
13515
13516 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13517 ("#GP Unexpected rc=%Rrc\n", rc));
13518 return rc;
13519}
13520
13521
13522/**
13523 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13524 * the exception reported in the VMX transient structure back into the VM.
13525 *
13526 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13527 * up-to-date.
13528 */
13529static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13530{
13531 RT_NOREF_PV(pMixedCtx);
13532 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13533#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13534 Assert(pVCpu->hm.s.fUsingDebugLoop);
13535#endif
13536
13537 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13538 hmR0VmxCheckExitDueToEventDelivery(). */
13539 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13540 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13541 AssertRCReturn(rc, rc);
13542 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13543
13544#ifdef DEBUG_ramshankar
13545 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13546 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13547 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13548#endif
13549
13550 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13551 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13552 return VINF_SUCCESS;
13553}
13554
13555
13556/**
13557 * VM-exit exception handler for \#PF (Page-fault exception).
13558 */
13559static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13560{
13561 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13562 PVM pVM = pVCpu->CTX_SUFF(pVM);
13563 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13564 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13565 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13566 AssertRCReturn(rc, rc);
13567
13568 if (!pVM->hm.s.fNestedPaging)
13569 { /* likely */ }
13570 else
13571 {
13572#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13573 Assert(pVCpu->hm.s.fUsingDebugLoop);
13574#endif
13575 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13576 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13577 {
13578 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13579 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13580 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13581 }
13582 else
13583 {
13584 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13585 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13586 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13587 }
13588 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13589 return rc;
13590 }
13591
13592 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13593 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13594 if (pVmxTransient->fVectoringPF)
13595 {
13596 Assert(pVCpu->hm.s.Event.fPending);
13597 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13598 }
13599
13600 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13601 AssertRCReturn(rc, rc);
13602
13603 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13604 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13605
13606 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13607 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13608 (RTGCPTR)pVmxTransient->uExitQualification);
13609
13610 Log4(("#PF: rc=%Rrc\n", rc));
13611 if (rc == VINF_SUCCESS)
13612 {
13613#if 0
13614 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13615 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13616 * memory? We don't update the whole state here... */
13617 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13618 | HM_CHANGED_GUEST_RSP
13619 | HM_CHANGED_GUEST_RFLAGS
13620 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13621#else
13622 /*
13623 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13624 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13625 */
13626 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13627 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13628#endif
13629 TRPMResetTrap(pVCpu);
13630 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13631 return rc;
13632 }
13633
13634 if (rc == VINF_EM_RAW_GUEST_TRAP)
13635 {
13636 if (!pVmxTransient->fVectoringDoublePF)
13637 {
13638 /* It's a guest page fault and needs to be reflected to the guest. */
13639 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13640 TRPMResetTrap(pVCpu);
13641 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13642 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13643 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13644 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13645 }
13646 else
13647 {
13648 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13649 TRPMResetTrap(pVCpu);
13650 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13651 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13652 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13653 }
13654
13655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13656 return VINF_SUCCESS;
13657 }
13658
13659 TRPMResetTrap(pVCpu);
13660 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13661 return rc;
13662}
13663
13664/** @} */
13665
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette