VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 61754

Last change on this file since 61754 was 61754, checked in by vboxsync, 9 years ago

VMM/HMVMXR0: Comment.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 579.7 KB
Line 
1/* $Id: HMVMXR0.cpp 61754 2016-06-17 15:37:42Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826/**
4827 * Sets up the appropriate function to run guest code.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu The cross context virtual CPU structure.
4831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832 * out-of-sync. Make sure to update the required fields
4833 * before using them.
4834 *
4835 * @remarks No-long-jump zone!!!
4836 */
4837static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838{
4839 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4840 {
4841#ifndef VBOX_ENABLE_64_BITS_GUESTS
4842 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4843#endif
4844 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4845#if HC_ARCH_BITS == 32
4846 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4847 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4850 {
4851 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4852 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4853 | HM_CHANGED_VMX_ENTRY_CTLS
4854 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4855 }
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4857
4858 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4859 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4860 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4861 }
4862#else
4863 /* 64-bit host. */
4864 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4865#endif
4866 }
4867 else
4868 {
4869 /* Guest is not in long mode, use the 32-bit handler. */
4870#if HC_ARCH_BITS == 32
4871 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4872 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4873 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4874 {
4875 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4876 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4877 | HM_CHANGED_VMX_ENTRY_CTLS
4878 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4879 }
4880# ifdef VBOX_ENABLE_64_BITS_GUESTS
4881 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}. */
4882 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4883 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4884 else
4885 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4886# else
4887 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4888# endif
4889#else
4890 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4891#endif
4892 }
4893 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4894 return VINF_SUCCESS;
4895}
4896
4897
4898/**
4899 * Wrapper for running the guest code in VT-x.
4900 *
4901 * @returns VBox status code, no informational status codes.
4902 * @param pVM The cross context VM structure.
4903 * @param pVCpu The cross context virtual CPU structure.
4904 * @param pCtx Pointer to the guest-CPU context.
4905 *
4906 * @remarks No-long-jump zone!!!
4907 */
4908DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4909{
4910 /*
4911 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4912 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4913 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4914 */
4915 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4916 /** @todo Add stats for resume vs launch. */
4917#ifdef VBOX_WITH_KERNEL_USING_XMM
4918 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4919#else
4920 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4921#endif
4922 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4923 return rc;
4924}
4925
4926
4927/**
4928 * Reports world-switch error and dumps some useful debug info.
4929 *
4930 * @param pVM The cross context VM structure.
4931 * @param pVCpu The cross context virtual CPU structure.
4932 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4933 * @param pCtx Pointer to the guest-CPU context.
4934 * @param pVmxTransient Pointer to the VMX transient structure (only
4935 * exitReason updated).
4936 */
4937static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4938{
4939 Assert(pVM);
4940 Assert(pVCpu);
4941 Assert(pCtx);
4942 Assert(pVmxTransient);
4943 HMVMX_ASSERT_PREEMPT_SAFE();
4944
4945 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4946 switch (rcVMRun)
4947 {
4948 case VERR_VMX_INVALID_VMXON_PTR:
4949 AssertFailed();
4950 break;
4951 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4952 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4953 {
4954 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4955 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4956 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4957 AssertRC(rc);
4958
4959 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4960 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4961 Cannot do it here as we may have been long preempted. */
4962
4963#ifdef VBOX_STRICT
4964 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4965 pVmxTransient->uExitReason));
4966 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4967 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4968 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4969 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4970 else
4971 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4972 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4973 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4974
4975 /* VMX control bits. */
4976 uint32_t u32Val;
4977 uint64_t u64Val;
4978 RTHCUINTREG uHCReg;
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5009 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5010 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5011 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5012 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5013 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5014 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5015 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5016 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5017 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5018 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5019 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5020 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5021
5022 /* Guest bits. */
5023 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5024 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5025 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5026 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5027 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5028 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5029 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5031
5032 /* Host bits. */
5033 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5034 Log4(("Host CR0 %#RHr\n", uHCReg));
5035 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5036 Log4(("Host CR3 %#RHr\n", uHCReg));
5037 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5038 Log4(("Host CR4 %#RHr\n", uHCReg));
5039
5040 RTGDTR HostGdtr;
5041 PCX86DESCHC pDesc;
5042 ASMGetGDTR(&HostGdtr);
5043 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5044 Log4(("Host CS %#08x\n", u32Val));
5045 if (u32Val < HostGdtr.cbGdt)
5046 {
5047 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5048 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5049 }
5050
5051 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5052 Log4(("Host DS %#08x\n", u32Val));
5053 if (u32Val < HostGdtr.cbGdt)
5054 {
5055 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5056 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5057 }
5058
5059 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5060 Log4(("Host ES %#08x\n", u32Val));
5061 if (u32Val < HostGdtr.cbGdt)
5062 {
5063 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5064 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5065 }
5066
5067 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5068 Log4(("Host FS %#08x\n", u32Val));
5069 if (u32Val < HostGdtr.cbGdt)
5070 {
5071 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5072 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5073 }
5074
5075 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5076 Log4(("Host GS %#08x\n", u32Val));
5077 if (u32Val < HostGdtr.cbGdt)
5078 {
5079 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5080 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5081 }
5082
5083 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5084 Log4(("Host SS %#08x\n", u32Val));
5085 if (u32Val < HostGdtr.cbGdt)
5086 {
5087 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5088 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5089 }
5090
5091 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5092 Log4(("Host TR %#08x\n", u32Val));
5093 if (u32Val < HostGdtr.cbGdt)
5094 {
5095 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5096 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5097 }
5098
5099 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5100 Log4(("Host TR Base %#RHv\n", uHCReg));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5102 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5103 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5104 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5105 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5106 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5108 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5109 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5110 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5111 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5112 Log4(("Host RSP %#RHv\n", uHCReg));
5113 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5114 Log4(("Host RIP %#RHv\n", uHCReg));
5115# if HC_ARCH_BITS == 64
5116 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5117 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5118 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5119 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5120 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5121 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5122# endif
5123#endif /* VBOX_STRICT */
5124 break;
5125 }
5126
5127 default:
5128 /* Impossible */
5129 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5130 break;
5131 }
5132 NOREF(pVM); NOREF(pCtx);
5133}
5134
5135
5136#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5137#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5138# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5139#endif
5140#ifdef VBOX_STRICT
5141static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5142{
5143 switch (idxField)
5144 {
5145 case VMX_VMCS_GUEST_RIP:
5146 case VMX_VMCS_GUEST_RSP:
5147 case VMX_VMCS_GUEST_SYSENTER_EIP:
5148 case VMX_VMCS_GUEST_SYSENTER_ESP:
5149 case VMX_VMCS_GUEST_GDTR_BASE:
5150 case VMX_VMCS_GUEST_IDTR_BASE:
5151 case VMX_VMCS_GUEST_CS_BASE:
5152 case VMX_VMCS_GUEST_DS_BASE:
5153 case VMX_VMCS_GUEST_ES_BASE:
5154 case VMX_VMCS_GUEST_FS_BASE:
5155 case VMX_VMCS_GUEST_GS_BASE:
5156 case VMX_VMCS_GUEST_SS_BASE:
5157 case VMX_VMCS_GUEST_LDTR_BASE:
5158 case VMX_VMCS_GUEST_TR_BASE:
5159 case VMX_VMCS_GUEST_CR3:
5160 return true;
5161 }
5162 return false;
5163}
5164
5165static bool hmR0VmxIsValidReadField(uint32_t idxField)
5166{
5167 switch (idxField)
5168 {
5169 /* Read-only fields. */
5170 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5171 return true;
5172 }
5173 /* Remaining readable fields should also be writable. */
5174 return hmR0VmxIsValidWriteField(idxField);
5175}
5176#endif /* VBOX_STRICT */
5177
5178
5179/**
5180 * Executes the specified handler in 64-bit mode.
5181 *
5182 * @returns VBox status code (no informational status codes).
5183 * @param pVM The cross context VM structure.
5184 * @param pVCpu The cross context virtual CPU structure.
5185 * @param pCtx Pointer to the guest CPU context.
5186 * @param enmOp The operation to perform.
5187 * @param cParams Number of parameters.
5188 * @param paParam Array of 32-bit parameters.
5189 */
5190VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5191 uint32_t cParams, uint32_t *paParam)
5192{
5193 NOREF(pCtx);
5194
5195 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5196 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5197 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5198 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5199
5200#ifdef VBOX_STRICT
5201 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5202 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5203
5204 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5205 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5206#endif
5207
5208 /* Disable interrupts. */
5209 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5210
5211#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5212 RTCPUID idHostCpu = RTMpCpuId();
5213 CPUMR0SetLApic(pVCpu, idHostCpu);
5214#endif
5215
5216 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5217 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5218
5219 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5220 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5221
5222 /* Leave VMX Root Mode. */
5223 VMXDisable();
5224
5225 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5226
5227 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5228 CPUMSetHyperEIP(pVCpu, enmOp);
5229 for (int i = (int)cParams - 1; i >= 0; i--)
5230 CPUMPushHyper(pVCpu, paParam[i]);
5231
5232 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5233
5234 /* Call the switcher. */
5235 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5236 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5237
5238 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5239 /* Make sure the VMX instructions don't cause #UD faults. */
5240 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5241
5242 /* Re-enter VMX Root Mode */
5243 int rc2 = VMXEnable(HCPhysCpuPage);
5244 if (RT_FAILURE(rc2))
5245 {
5246 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5247 ASMSetFlags(fOldEFlags);
5248 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5249 return rc2;
5250 }
5251
5252 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5253 AssertRC(rc2);
5254 Assert(!(ASMGetFlags() & X86_EFL_IF));
5255 ASMSetFlags(fOldEFlags);
5256 return rc;
5257}
5258
5259
5260/**
5261 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5262 * supporting 64-bit guests.
5263 *
5264 * @returns VBox status code.
5265 * @param fResume Whether to VMLAUNCH or VMRESUME.
5266 * @param pCtx Pointer to the guest-CPU context.
5267 * @param pCache Pointer to the VMCS cache.
5268 * @param pVM The cross context VM structure.
5269 * @param pVCpu The cross context virtual CPU structure.
5270 */
5271DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5272{
5273 NOREF(fResume);
5274
5275 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5276 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5277
5278#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5279 pCache->uPos = 1;
5280 pCache->interPD = PGMGetInterPaeCR3(pVM);
5281 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5282#endif
5283
5284#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5285 pCache->TestIn.HCPhysCpuPage = 0;
5286 pCache->TestIn.HCPhysVmcs = 0;
5287 pCache->TestIn.pCache = 0;
5288 pCache->TestOut.HCPhysVmcs = 0;
5289 pCache->TestOut.pCache = 0;
5290 pCache->TestOut.pCtx = 0;
5291 pCache->TestOut.eflags = 0;
5292#else
5293 NOREF(pCache);
5294#endif
5295
5296 uint32_t aParam[10];
5297 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5298 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5299 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5300 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5301 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5302 aParam[5] = 0;
5303 aParam[6] = VM_RC_ADDR(pVM, pVM);
5304 aParam[7] = 0;
5305 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5306 aParam[9] = 0;
5307
5308#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5309 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5310 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5311#endif
5312 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5313
5314#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5315 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5316 Assert(pCtx->dr[4] == 10);
5317 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5318#endif
5319
5320#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5321 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5322 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5323 pVCpu->hm.s.vmx.HCPhysVmcs));
5324 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5325 pCache->TestOut.HCPhysVmcs));
5326 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5327 pCache->TestOut.pCache));
5328 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5329 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5330 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5331 pCache->TestOut.pCtx));
5332 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5333#endif
5334 return rc;
5335}
5336
5337
5338/**
5339 * Initialize the VMCS-Read cache.
5340 *
5341 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5342 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5343 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5344 * (those that have a 32-bit FULL & HIGH part).
5345 *
5346 * @returns VBox status code.
5347 * @param pVM The cross context VM structure.
5348 * @param pVCpu The cross context virtual CPU structure.
5349 */
5350static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5351{
5352#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5353{ \
5354 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5355 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5356 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5357 ++cReadFields; \
5358}
5359
5360 AssertPtr(pVM);
5361 AssertPtr(pVCpu);
5362 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5363 uint32_t cReadFields = 0;
5364
5365 /*
5366 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5367 * and serve to indicate exceptions to the rules.
5368 */
5369
5370 /* Guest-natural selector base fields. */
5371#if 0
5372 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5375#endif
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5388#if 0
5389 /* Unused natural width guest-state fields. */
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5392#endif
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5395
5396 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5397#if 0
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5407#endif
5408
5409 /* Natural width guest-state fields. */
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5411#if 0
5412 /* Currently unused field. */
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5414#endif
5415
5416 if (pVM->hm.s.fNestedPaging)
5417 {
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5419 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5420 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5421 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5422 }
5423 else
5424 {
5425 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5426 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5427 }
5428
5429#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5430 return VINF_SUCCESS;
5431}
5432
5433
5434/**
5435 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5436 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5437 * darwin, running 64-bit guests).
5438 *
5439 * @returns VBox status code.
5440 * @param pVCpu The cross context virtual CPU structure.
5441 * @param idxField The VMCS field encoding.
5442 * @param u64Val 16, 32 or 64-bit value.
5443 */
5444VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5445{
5446 int rc;
5447 switch (idxField)
5448 {
5449 /*
5450 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5451 */
5452 /* 64-bit Control fields. */
5453 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5454 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5455 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5456 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5457 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5458 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5459 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5460 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5461 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5462 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5463 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5464 case VMX_VMCS64_CTRL_EPTP_FULL:
5465 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5466 /* 64-bit Guest-state fields. */
5467 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5468 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5469 case VMX_VMCS64_GUEST_PAT_FULL:
5470 case VMX_VMCS64_GUEST_EFER_FULL:
5471 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5472 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5473 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5474 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5475 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5476 /* 64-bit Host-state fields. */
5477 case VMX_VMCS64_HOST_PAT_FULL:
5478 case VMX_VMCS64_HOST_EFER_FULL:
5479 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5480 {
5481 rc = VMXWriteVmcs32(idxField, u64Val);
5482 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5483 break;
5484 }
5485
5486 /*
5487 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5488 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5489 */
5490 /* Natural-width Guest-state fields. */
5491 case VMX_VMCS_GUEST_CR3:
5492 case VMX_VMCS_GUEST_ES_BASE:
5493 case VMX_VMCS_GUEST_CS_BASE:
5494 case VMX_VMCS_GUEST_SS_BASE:
5495 case VMX_VMCS_GUEST_DS_BASE:
5496 case VMX_VMCS_GUEST_FS_BASE:
5497 case VMX_VMCS_GUEST_GS_BASE:
5498 case VMX_VMCS_GUEST_LDTR_BASE:
5499 case VMX_VMCS_GUEST_TR_BASE:
5500 case VMX_VMCS_GUEST_GDTR_BASE:
5501 case VMX_VMCS_GUEST_IDTR_BASE:
5502 case VMX_VMCS_GUEST_RSP:
5503 case VMX_VMCS_GUEST_RIP:
5504 case VMX_VMCS_GUEST_SYSENTER_ESP:
5505 case VMX_VMCS_GUEST_SYSENTER_EIP:
5506 {
5507 if (!(u64Val >> 32))
5508 {
5509 /* If this field is 64-bit, VT-x will zero out the top bits. */
5510 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5511 }
5512 else
5513 {
5514 /* Assert that only the 32->64 switcher case should ever come here. */
5515 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5516 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5517 }
5518 break;
5519 }
5520
5521 default:
5522 {
5523 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5524 rc = VERR_INVALID_PARAMETER;
5525 break;
5526 }
5527 }
5528 AssertRCReturn(rc, rc);
5529 return rc;
5530}
5531
5532
5533/**
5534 * Queue up a VMWRITE by using the VMCS write cache.
5535 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5536 *
5537 * @param pVCpu The cross context virtual CPU structure.
5538 * @param idxField The VMCS field encoding.
5539 * @param u64Val 16, 32 or 64-bit value.
5540 */
5541VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5542{
5543 AssertPtr(pVCpu);
5544 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5545
5546 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5547 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5548
5549 /* Make sure there are no duplicates. */
5550 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5551 {
5552 if (pCache->Write.aField[i] == idxField)
5553 {
5554 pCache->Write.aFieldVal[i] = u64Val;
5555 return VINF_SUCCESS;
5556 }
5557 }
5558
5559 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5560 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5561 pCache->Write.cValidEntries++;
5562 return VINF_SUCCESS;
5563}
5564#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5565
5566
5567/**
5568 * Sets up the usage of TSC-offsetting and updates the VMCS.
5569 *
5570 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5571 * VMX preemption timer.
5572 *
5573 * @returns VBox status code.
5574 * @param pVM The cross context VM structure.
5575 * @param pVCpu The cross context virtual CPU structure.
5576 *
5577 * @remarks No-long-jump zone!!!
5578 */
5579static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5580{
5581 int rc;
5582 bool fOffsettedTsc;
5583 bool fParavirtTsc;
5584 if (pVM->hm.s.vmx.fUsePreemptTimer)
5585 {
5586 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5587 &fOffsettedTsc, &fParavirtTsc);
5588
5589 /* Make sure the returned values have sane upper and lower boundaries. */
5590 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5591 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5592 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5593 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5594
5595 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5596 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5597 }
5598 else
5599 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5600
5601 /** @todo later optimize this to be done elsewhere and not before every
5602 * VM-entry. */
5603 if (fParavirtTsc)
5604 {
5605 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5606 information before every VM-entry, hence disable it for performance sake. */
5607#if 0
5608 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5609 AssertRC(rc);
5610#endif
5611 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5612 }
5613
5614 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5615 {
5616 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5617 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5618
5619 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5620 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5621 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5622 }
5623 else
5624 {
5625 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5626 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5627 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5628 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5629 }
5630}
5631
5632
5633/**
5634 * Determines if an exception is a contributory exception.
5635 *
5636 * Contributory exceptions are ones which can cause double-faults unless the
5637 * original exception was a benign exception. Page-fault is intentionally not
5638 * included here as it's a conditional contributory exception.
5639 *
5640 * @returns true if the exception is contributory, false otherwise.
5641 * @param uVector The exception vector.
5642 */
5643DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5644{
5645 switch (uVector)
5646 {
5647 case X86_XCPT_GP:
5648 case X86_XCPT_SS:
5649 case X86_XCPT_NP:
5650 case X86_XCPT_TS:
5651 case X86_XCPT_DE:
5652 return true;
5653 default:
5654 break;
5655 }
5656 return false;
5657}
5658
5659
5660/**
5661 * Sets an event as a pending event to be injected into the guest.
5662 *
5663 * @param pVCpu The cross context virtual CPU structure.
5664 * @param u32IntInfo The VM-entry interruption-information field.
5665 * @param cbInstr The VM-entry instruction length in bytes (for software
5666 * interrupts, exceptions and privileged software
5667 * exceptions).
5668 * @param u32ErrCode The VM-entry exception error code.
5669 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5670 * page-fault.
5671 *
5672 * @remarks Statistics counter assumes this is a guest event being injected or
5673 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5674 * always incremented.
5675 */
5676DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5677 RTGCUINTPTR GCPtrFaultAddress)
5678{
5679 Assert(!pVCpu->hm.s.Event.fPending);
5680 pVCpu->hm.s.Event.fPending = true;
5681 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5682 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5683 pVCpu->hm.s.Event.cbInstr = cbInstr;
5684 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5685}
5686
5687
5688/**
5689 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5690 *
5691 * @param pVCpu The cross context virtual CPU structure.
5692 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5693 * out-of-sync. Make sure to update the required fields
5694 * before using them.
5695 */
5696DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5697{
5698 NOREF(pMixedCtx);
5699 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5700 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5701 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5702 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5703}
5704
5705
5706/**
5707 * Handle a condition that occurred while delivering an event through the guest
5708 * IDT.
5709 *
5710 * @returns Strict VBox status code (i.e. informational status codes too).
5711 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5712 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5713 * to continue execution of the guest which will delivery the \#DF.
5714 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5715 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5716 *
5717 * @param pVCpu The cross context virtual CPU structure.
5718 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5719 * out-of-sync. Make sure to update the required fields
5720 * before using them.
5721 * @param pVmxTransient Pointer to the VMX transient structure.
5722 *
5723 * @remarks No-long-jump zone!!!
5724 */
5725static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5726{
5727 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5728
5729 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5730 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5731
5732 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5733 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5734 {
5735 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5736 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5737
5738 typedef enum
5739 {
5740 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5741 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5742 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5743 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5744 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5745 } VMXREFLECTXCPT;
5746
5747 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5748 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5749 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5750 {
5751 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5752 {
5753 enmReflect = VMXREFLECTXCPT_XCPT;
5754#ifdef VBOX_STRICT
5755 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5756 && uExitVector == X86_XCPT_PF)
5757 {
5758 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5759 }
5760#endif
5761 if ( uExitVector == X86_XCPT_PF
5762 && uIdtVector == X86_XCPT_PF)
5763 {
5764 pVmxTransient->fVectoringDoublePF = true;
5765 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5766 }
5767 else if ( uExitVector == X86_XCPT_AC
5768 && uIdtVector == X86_XCPT_AC)
5769 {
5770 enmReflect = VMXREFLECTXCPT_HANG;
5771 Log4(("IDT: Nested #AC - Bad guest\n"));
5772 }
5773 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5774 && hmR0VmxIsContributoryXcpt(uExitVector)
5775 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5776 || uIdtVector == X86_XCPT_PF))
5777 {
5778 enmReflect = VMXREFLECTXCPT_DF;
5779 }
5780 else if (uIdtVector == X86_XCPT_DF)
5781 enmReflect = VMXREFLECTXCPT_TF;
5782 }
5783 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5784 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5785 {
5786 /*
5787 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5788 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5789 */
5790 enmReflect = VMXREFLECTXCPT_XCPT;
5791
5792 if (uExitVector == X86_XCPT_PF)
5793 {
5794 pVmxTransient->fVectoringPF = true;
5795 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5796 }
5797 }
5798 }
5799 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5800 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5801 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5802 {
5803 /*
5804 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5805 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5806 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5807 */
5808 enmReflect = VMXREFLECTXCPT_XCPT;
5809 }
5810
5811 /*
5812 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5813 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5814 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5815 *
5816 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5817 */
5818 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5819 && enmReflect == VMXREFLECTXCPT_XCPT
5820 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5821 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5822 {
5823 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5824 }
5825
5826 switch (enmReflect)
5827 {
5828 case VMXREFLECTXCPT_XCPT:
5829 {
5830 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5831 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5832 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5833
5834 uint32_t u32ErrCode = 0;
5835 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5836 {
5837 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5838 AssertRCReturn(rc2, rc2);
5839 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5840 }
5841
5842 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5843 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5844 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5845 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5846 rcStrict = VINF_SUCCESS;
5847 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5848 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5849
5850 break;
5851 }
5852
5853 case VMXREFLECTXCPT_DF:
5854 {
5855 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5856 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5857 rcStrict = VINF_HM_DOUBLE_FAULT;
5858 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5859 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5860
5861 break;
5862 }
5863
5864 case VMXREFLECTXCPT_TF:
5865 {
5866 rcStrict = VINF_EM_RESET;
5867 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5868 uExitVector));
5869 break;
5870 }
5871
5872 case VMXREFLECTXCPT_HANG:
5873 {
5874 rcStrict = VERR_EM_GUEST_CPU_HANG;
5875 break;
5876 }
5877
5878 default:
5879 Assert(rcStrict == VINF_SUCCESS);
5880 break;
5881 }
5882 }
5883 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5884 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5885 && uExitVector != X86_XCPT_DF
5886 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5887 {
5888 /*
5889 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5890 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5891 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5892 */
5893 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5894 {
5895 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5896 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5897 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5898 }
5899 }
5900
5901 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5902 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5903 return rcStrict;
5904}
5905
5906
5907/**
5908 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5909 *
5910 * @returns VBox status code.
5911 * @param pVCpu The cross context virtual CPU structure.
5912 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5913 * out-of-sync. Make sure to update the required fields
5914 * before using them.
5915 *
5916 * @remarks No-long-jump zone!!!
5917 */
5918static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5919{
5920 NOREF(pMixedCtx);
5921
5922 /*
5923 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5924 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5925 */
5926 VMMRZCallRing3Disable(pVCpu);
5927 HM_DISABLE_PREEMPT();
5928
5929 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5930 {
5931 uint32_t uVal = 0;
5932 uint32_t uShadow = 0;
5933 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5934 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5935 AssertRCReturn(rc, rc);
5936
5937 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5938 CPUMSetGuestCR0(pVCpu, uVal);
5939 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5940 }
5941
5942 HM_RESTORE_PREEMPT();
5943 VMMRZCallRing3Enable(pVCpu);
5944 return VINF_SUCCESS;
5945}
5946
5947
5948/**
5949 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5950 *
5951 * @returns VBox status code.
5952 * @param pVCpu The cross context virtual CPU structure.
5953 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5954 * out-of-sync. Make sure to update the required fields
5955 * before using them.
5956 *
5957 * @remarks No-long-jump zone!!!
5958 */
5959static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5960{
5961 NOREF(pMixedCtx);
5962
5963 int rc = VINF_SUCCESS;
5964 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5965 {
5966 uint32_t uVal = 0;
5967 uint32_t uShadow = 0;
5968 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5969 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5970 AssertRCReturn(rc, rc);
5971
5972 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5973 CPUMSetGuestCR4(pVCpu, uVal);
5974 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5975 }
5976 return rc;
5977}
5978
5979
5980/**
5981 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5982 *
5983 * @returns VBox status code.
5984 * @param pVCpu The cross context virtual CPU structure.
5985 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5986 * out-of-sync. Make sure to update the required fields
5987 * before using them.
5988 *
5989 * @remarks No-long-jump zone!!!
5990 */
5991static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5992{
5993 int rc = VINF_SUCCESS;
5994 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5995 {
5996 uint64_t u64Val = 0;
5997 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5998 AssertRCReturn(rc, rc);
5999
6000 pMixedCtx->rip = u64Val;
6001 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6002 }
6003 return rc;
6004}
6005
6006
6007/**
6008 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6009 *
6010 * @returns VBox status code.
6011 * @param pVCpu The cross context virtual CPU structure.
6012 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6013 * out-of-sync. Make sure to update the required fields
6014 * before using them.
6015 *
6016 * @remarks No-long-jump zone!!!
6017 */
6018static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6019{
6020 int rc = VINF_SUCCESS;
6021 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6022 {
6023 uint64_t u64Val = 0;
6024 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6025 AssertRCReturn(rc, rc);
6026
6027 pMixedCtx->rsp = u64Val;
6028 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6029 }
6030 return rc;
6031}
6032
6033
6034/**
6035 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6036 *
6037 * @returns VBox status code.
6038 * @param pVCpu The cross context virtual CPU structure.
6039 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6040 * out-of-sync. Make sure to update the required fields
6041 * before using them.
6042 *
6043 * @remarks No-long-jump zone!!!
6044 */
6045static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046{
6047 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6048 {
6049 uint32_t uVal = 0;
6050 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6051 AssertRCReturn(rc, rc);
6052
6053 pMixedCtx->eflags.u32 = uVal;
6054 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6055 {
6056 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6057 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6058
6059 pMixedCtx->eflags.Bits.u1VM = 0;
6060 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6061 }
6062
6063 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6064 }
6065 return VINF_SUCCESS;
6066}
6067
6068
6069/**
6070 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6071 * guest-CPU context.
6072 */
6073DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6074{
6075 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6076 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6077 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6078 return rc;
6079}
6080
6081
6082/**
6083 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6084 * from the guest-state area in the VMCS.
6085 *
6086 * @param pVCpu The cross context virtual CPU structure.
6087 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6088 * out-of-sync. Make sure to update the required fields
6089 * before using them.
6090 *
6091 * @remarks No-long-jump zone!!!
6092 */
6093static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6094{
6095 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6096 {
6097 uint32_t uIntrState = 0;
6098 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6099 AssertRC(rc);
6100
6101 if (!uIntrState)
6102 {
6103 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6104 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6105
6106 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6107 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6108 }
6109 else
6110 {
6111 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6112 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6113 {
6114 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6115 AssertRC(rc);
6116 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6117 AssertRC(rc);
6118
6119 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6120 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6121 }
6122 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6123 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6124
6125 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6126 {
6127 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6128 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6129 }
6130 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6131 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6132 }
6133
6134 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6135 }
6136}
6137
6138
6139/**
6140 * Saves the guest's activity state.
6141 *
6142 * @returns VBox status code.
6143 * @param pVCpu The cross context virtual CPU structure.
6144 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6145 * out-of-sync. Make sure to update the required fields
6146 * before using them.
6147 *
6148 * @remarks No-long-jump zone!!!
6149 */
6150static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6151{
6152 NOREF(pMixedCtx);
6153 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6154 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6155 return VINF_SUCCESS;
6156}
6157
6158
6159/**
6160 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6161 * the current VMCS into the guest-CPU context.
6162 *
6163 * @returns VBox status code.
6164 * @param pVCpu The cross context virtual CPU structure.
6165 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6166 * out-of-sync. Make sure to update the required fields
6167 * before using them.
6168 *
6169 * @remarks No-long-jump zone!!!
6170 */
6171static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6172{
6173 int rc = VINF_SUCCESS;
6174 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6175 {
6176 uint32_t u32Val = 0;
6177 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6178 pMixedCtx->SysEnter.cs = u32Val;
6179 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6180 }
6181
6182 uint64_t u64Val = 0;
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6184 {
6185 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6186 pMixedCtx->SysEnter.eip = u64Val;
6187 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6188 }
6189 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6190 {
6191 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6192 pMixedCtx->SysEnter.esp = u64Val;
6193 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6194 }
6195 return rc;
6196}
6197
6198
6199/**
6200 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6201 * the CPU back into the guest-CPU context.
6202 *
6203 * @returns VBox status code.
6204 * @param pVCpu The cross context virtual CPU structure.
6205 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6206 * out-of-sync. Make sure to update the required fields
6207 * before using them.
6208 *
6209 * @remarks No-long-jump zone!!!
6210 */
6211static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6212{
6213 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6214 VMMRZCallRing3Disable(pVCpu);
6215 HM_DISABLE_PREEMPT();
6216
6217 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6218 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6219 {
6220 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6221 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6222 }
6223
6224 HM_RESTORE_PREEMPT();
6225 VMMRZCallRing3Enable(pVCpu);
6226
6227 return VINF_SUCCESS;
6228}
6229
6230
6231/**
6232 * Saves the auto load/store'd guest MSRs from the current VMCS into
6233 * the guest-CPU context.
6234 *
6235 * @returns VBox status code.
6236 * @param pVCpu The cross context virtual CPU structure.
6237 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6238 * out-of-sync. Make sure to update the required fields
6239 * before using them.
6240 *
6241 * @remarks No-long-jump zone!!!
6242 */
6243static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6244{
6245 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6246 return VINF_SUCCESS;
6247
6248 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6249 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6250 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6251 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6252 {
6253 switch (pMsr->u32Msr)
6254 {
6255 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6256 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6257 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6258 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6259 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6260 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6261 break;
6262
6263 default:
6264 {
6265 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6266 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6267 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6268 }
6269 }
6270 }
6271
6272 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6273 return VINF_SUCCESS;
6274}
6275
6276
6277/**
6278 * Saves the guest control registers from the current VMCS into the guest-CPU
6279 * context.
6280 *
6281 * @returns VBox status code.
6282 * @param pVCpu The cross context virtual CPU structure.
6283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6284 * out-of-sync. Make sure to update the required fields
6285 * before using them.
6286 *
6287 * @remarks No-long-jump zone!!!
6288 */
6289static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6290{
6291 /* Guest CR0. Guest FPU. */
6292 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6293 AssertRCReturn(rc, rc);
6294
6295 /* Guest CR4. */
6296 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6297 AssertRCReturn(rc, rc);
6298
6299 /* Guest CR2 - updated always during the world-switch or in #PF. */
6300 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6301 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6302 {
6303 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6304 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6305
6306 PVM pVM = pVCpu->CTX_SUFF(pVM);
6307 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6308 || ( pVM->hm.s.fNestedPaging
6309 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6310 {
6311 uint64_t u64Val = 0;
6312 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6313 if (pMixedCtx->cr3 != u64Val)
6314 {
6315 CPUMSetGuestCR3(pVCpu, u64Val);
6316 if (VMMRZCallRing3IsEnabled(pVCpu))
6317 {
6318 PGMUpdateCR3(pVCpu, u64Val);
6319 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6320 }
6321 else
6322 {
6323 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6324 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6325 }
6326 }
6327
6328 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6329 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6330 {
6331 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6332 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6333 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6334 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6335 AssertRCReturn(rc, rc);
6336
6337 if (VMMRZCallRing3IsEnabled(pVCpu))
6338 {
6339 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6340 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6341 }
6342 else
6343 {
6344 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6345 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6346 }
6347 }
6348 }
6349
6350 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6351 }
6352
6353 /*
6354 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6355 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6356 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6357 *
6358 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6359 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6360 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6361 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6362 *
6363 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6364 */
6365 if (VMMRZCallRing3IsEnabled(pVCpu))
6366 {
6367 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6368 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6369
6370 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6371 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6372
6373 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6374 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6375 }
6376
6377 return rc;
6378}
6379
6380
6381/**
6382 * Reads a guest segment register from the current VMCS into the guest-CPU
6383 * context.
6384 *
6385 * @returns VBox status code.
6386 * @param pVCpu The cross context virtual CPU structure.
6387 * @param idxSel Index of the selector in the VMCS.
6388 * @param idxLimit Index of the segment limit in the VMCS.
6389 * @param idxBase Index of the segment base in the VMCS.
6390 * @param idxAccess Index of the access rights of the segment in the VMCS.
6391 * @param pSelReg Pointer to the segment selector.
6392 *
6393 * @remarks No-long-jump zone!!!
6394 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6395 * macro as that takes care of whether to read from the VMCS cache or
6396 * not.
6397 */
6398DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6399 PCPUMSELREG pSelReg)
6400{
6401 NOREF(pVCpu);
6402
6403 uint32_t u32Val = 0;
6404 int rc = VMXReadVmcs32(idxSel, &u32Val);
6405 AssertRCReturn(rc, rc);
6406 pSelReg->Sel = (uint16_t)u32Val;
6407 pSelReg->ValidSel = (uint16_t)u32Val;
6408 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6409
6410 rc = VMXReadVmcs32(idxLimit, &u32Val);
6411 AssertRCReturn(rc, rc);
6412 pSelReg->u32Limit = u32Val;
6413
6414 uint64_t u64Val = 0;
6415 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6416 AssertRCReturn(rc, rc);
6417 pSelReg->u64Base = u64Val;
6418
6419 rc = VMXReadVmcs32(idxAccess, &u32Val);
6420 AssertRCReturn(rc, rc);
6421 pSelReg->Attr.u = u32Val;
6422
6423 /*
6424 * If VT-x marks the segment as unusable, most other bits remain undefined:
6425 * - For CS the L, D and G bits have meaning.
6426 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6427 * - For the remaining data segments no bits are defined.
6428 *
6429 * The present bit and the unusable bit has been observed to be set at the
6430 * same time (the selector was supposed to be invalid as we started executing
6431 * a V8086 interrupt in ring-0).
6432 *
6433 * What should be important for the rest of the VBox code, is that the P bit is
6434 * cleared. Some of the other VBox code recognizes the unusable bit, but
6435 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6436 * safe side here, we'll strip off P and other bits we don't care about. If
6437 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6438 *
6439 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6440 */
6441 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6442 {
6443 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6444
6445 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6446 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6447 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6448
6449 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6450#ifdef DEBUG_bird
6451 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6452 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6453 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6454#endif
6455 }
6456 return VINF_SUCCESS;
6457}
6458
6459
6460#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6461# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6462 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6463 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6464#else
6465# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6466 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6467 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6468#endif
6469
6470
6471/**
6472 * Saves the guest segment registers from the current VMCS into the guest-CPU
6473 * context.
6474 *
6475 * @returns VBox status code.
6476 * @param pVCpu The cross context virtual CPU structure.
6477 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6478 * out-of-sync. Make sure to update the required fields
6479 * before using them.
6480 *
6481 * @remarks No-long-jump zone!!!
6482 */
6483static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6484{
6485 /* Guest segment registers. */
6486 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6487 {
6488 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6489 AssertRCReturn(rc, rc);
6490
6491 rc = VMXLOCAL_READ_SEG(CS, cs);
6492 rc |= VMXLOCAL_READ_SEG(SS, ss);
6493 rc |= VMXLOCAL_READ_SEG(DS, ds);
6494 rc |= VMXLOCAL_READ_SEG(ES, es);
6495 rc |= VMXLOCAL_READ_SEG(FS, fs);
6496 rc |= VMXLOCAL_READ_SEG(GS, gs);
6497 AssertRCReturn(rc, rc);
6498
6499 /* Restore segment attributes for real-on-v86 mode hack. */
6500 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6501 {
6502 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6503 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6504 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6505 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6506 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6507 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6508 }
6509 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6510 }
6511
6512 return VINF_SUCCESS;
6513}
6514
6515
6516/**
6517 * Saves the guest descriptor table registers and task register from the current
6518 * VMCS into the guest-CPU context.
6519 *
6520 * @returns VBox status code.
6521 * @param pVCpu The cross context virtual CPU structure.
6522 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6523 * out-of-sync. Make sure to update the required fields
6524 * before using them.
6525 *
6526 * @remarks No-long-jump zone!!!
6527 */
6528static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6529{
6530 int rc = VINF_SUCCESS;
6531
6532 /* Guest LDTR. */
6533 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6534 {
6535 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6536 AssertRCReturn(rc, rc);
6537 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6538 }
6539
6540 /* Guest GDTR. */
6541 uint64_t u64Val = 0;
6542 uint32_t u32Val = 0;
6543 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6544 {
6545 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6546 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6547 pMixedCtx->gdtr.pGdt = u64Val;
6548 pMixedCtx->gdtr.cbGdt = u32Val;
6549 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6550 }
6551
6552 /* Guest IDTR. */
6553 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6554 {
6555 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6556 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6557 pMixedCtx->idtr.pIdt = u64Val;
6558 pMixedCtx->idtr.cbIdt = u32Val;
6559 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6560 }
6561
6562 /* Guest TR. */
6563 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6564 {
6565 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6566 AssertRCReturn(rc, rc);
6567
6568 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6569 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6570 {
6571 rc = VMXLOCAL_READ_SEG(TR, tr);
6572 AssertRCReturn(rc, rc);
6573 }
6574 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6575 }
6576 return rc;
6577}
6578
6579#undef VMXLOCAL_READ_SEG
6580
6581
6582/**
6583 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6584 * context.
6585 *
6586 * @returns VBox status code.
6587 * @param pVCpu The cross context virtual CPU structure.
6588 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6589 * out-of-sync. Make sure to update the required fields
6590 * before using them.
6591 *
6592 * @remarks No-long-jump zone!!!
6593 */
6594static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6595{
6596 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6597 {
6598 if (!pVCpu->hm.s.fUsingHyperDR7)
6599 {
6600 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6601 uint32_t u32Val;
6602 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6603 pMixedCtx->dr[7] = u32Val;
6604 }
6605
6606 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6607 }
6608 return VINF_SUCCESS;
6609}
6610
6611
6612/**
6613 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6614 *
6615 * @returns VBox status code.
6616 * @param pVCpu The cross context virtual CPU structure.
6617 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6618 * out-of-sync. Make sure to update the required fields
6619 * before using them.
6620 *
6621 * @remarks No-long-jump zone!!!
6622 */
6623static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6624{
6625 NOREF(pMixedCtx);
6626
6627 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6628 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6629 return VINF_SUCCESS;
6630}
6631
6632
6633/**
6634 * Saves the entire guest state from the currently active VMCS into the
6635 * guest-CPU context.
6636 *
6637 * This essentially VMREADs all guest-data.
6638 *
6639 * @returns VBox status code.
6640 * @param pVCpu The cross context virtual CPU structure.
6641 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6642 * out-of-sync. Make sure to update the required fields
6643 * before using them.
6644 */
6645static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6646{
6647 Assert(pVCpu);
6648 Assert(pMixedCtx);
6649
6650 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6651 return VINF_SUCCESS;
6652
6653 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6654 again on the ring-3 callback path, there is no real need to. */
6655 if (VMMRZCallRing3IsEnabled(pVCpu))
6656 VMMR0LogFlushDisable(pVCpu);
6657 else
6658 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6659 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6660
6661 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6662 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6663
6664 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6665 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6666
6667 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6668 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6669
6670 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6671 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6672
6673 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6674 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6675
6676 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6677 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6678
6679 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6686 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6687
6688 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6692 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6693 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6694
6695 if (VMMRZCallRing3IsEnabled(pVCpu))
6696 VMMR0LogFlushEnable(pVCpu);
6697
6698 return VINF_SUCCESS;
6699}
6700
6701
6702/**
6703 * Saves basic guest registers needed for IEM instruction execution.
6704 *
6705 * @returns VBox status code (OR-able).
6706 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6707 * @param pMixedCtx Pointer to the CPU context of the guest.
6708 * @param fMemory Whether the instruction being executed operates on
6709 * memory or not. Only CR0 is synced up if clear.
6710 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6711 */
6712static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6713{
6714 /*
6715 * We assume all general purpose registers other than RSP are available.
6716 *
6717 * RIP is a must, as it will be incremented or otherwise changed.
6718 *
6719 * RFLAGS are always required to figure the CPL.
6720 *
6721 * RSP isn't always required, however it's a GPR, so frequently required.
6722 *
6723 * SS and CS are the only segment register needed if IEM doesn't do memory
6724 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6725 *
6726 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6727 * be required for memory accesses.
6728 *
6729 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6730 */
6731 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6732 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6733 if (fNeedRsp)
6734 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6735 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6736 if (!fMemory)
6737 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6738 else
6739 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6740 AssertRCReturn(rc, rc);
6741 return rc;
6742}
6743
6744
6745/**
6746 * Ensures that we've got a complete basic guest-context.
6747 *
6748 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6749 * is for the interpreter.
6750 *
6751 * @returns VBox status code.
6752 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6753 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6754 * needing to be synced in.
6755 * @thread EMT(pVCpu)
6756 */
6757VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6758{
6759 /* Note! Since this is only applicable to VT-x, the implementation is placed
6760 in the VT-x part of the sources instead of the generic stuff. */
6761 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6762 {
6763 /* For now, imply that the caller might change everything too. */
6764 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6765 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6766 }
6767 return VINF_SUCCESS;
6768}
6769
6770
6771/**
6772 * Check per-VM and per-VCPU force flag actions that require us to go back to
6773 * ring-3 for one reason or another.
6774 *
6775 * @returns Strict VBox status code (i.e. informational status codes too)
6776 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6777 * ring-3.
6778 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6779 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6780 * interrupts)
6781 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6782 * all EMTs to be in ring-3.
6783 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6784 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6785 * to the EM loop.
6786 *
6787 * @param pVM The cross context VM structure.
6788 * @param pVCpu The cross context virtual CPU structure.
6789 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6790 * out-of-sync. Make sure to update the required fields
6791 * before using them.
6792 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6793 */
6794static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6795{
6796 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6797
6798 /*
6799 * Anything pending? Should be more likely than not if we're doing a good job.
6800 */
6801 if ( !fStepping
6802 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6803 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6804 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6805 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6806 return VINF_SUCCESS;
6807
6808 /* We need the control registers now, make sure the guest-CPU context is updated. */
6809 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6810 AssertRCReturn(rc3, rc3);
6811
6812 /* Pending HM CR3 sync. */
6813 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6814 {
6815 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6816 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6817 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6818 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6819 }
6820
6821 /* Pending HM PAE PDPEs. */
6822 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6823 {
6824 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6825 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6826 }
6827
6828 /* Pending PGM C3 sync. */
6829 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6830 {
6831 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6832 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6833 if (rcStrict2 != VINF_SUCCESS)
6834 {
6835 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6836 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6837 return rcStrict2;
6838 }
6839 }
6840
6841 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6842 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6843 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6844 {
6845 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6846 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6847 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6848 return rc2;
6849 }
6850
6851 /* Pending VM request packets, such as hardware interrupts. */
6852 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6853 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6854 {
6855 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6856 return VINF_EM_PENDING_REQUEST;
6857 }
6858
6859 /* Pending PGM pool flushes. */
6860 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6861 {
6862 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6863 return VINF_PGM_POOL_FLUSH_PENDING;
6864 }
6865
6866 /* Pending DMA requests. */
6867 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6868 {
6869 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6870 return VINF_EM_RAW_TO_R3;
6871 }
6872
6873 return VINF_SUCCESS;
6874}
6875
6876
6877/**
6878 * Converts any TRPM trap into a pending HM event. This is typically used when
6879 * entering from ring-3 (not longjmp returns).
6880 *
6881 * @param pVCpu The cross context virtual CPU structure.
6882 */
6883static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6884{
6885 Assert(TRPMHasTrap(pVCpu));
6886 Assert(!pVCpu->hm.s.Event.fPending);
6887
6888 uint8_t uVector;
6889 TRPMEVENT enmTrpmEvent;
6890 RTGCUINT uErrCode;
6891 RTGCUINTPTR GCPtrFaultAddress;
6892 uint8_t cbInstr;
6893
6894 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6895 AssertRC(rc);
6896
6897 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6898 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6899 if (enmTrpmEvent == TRPM_TRAP)
6900 {
6901 switch (uVector)
6902 {
6903 case X86_XCPT_NMI:
6904 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6905 break;
6906
6907 case X86_XCPT_BP:
6908 case X86_XCPT_OF:
6909 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6910 break;
6911
6912 case X86_XCPT_PF:
6913 case X86_XCPT_DF:
6914 case X86_XCPT_TS:
6915 case X86_XCPT_NP:
6916 case X86_XCPT_SS:
6917 case X86_XCPT_GP:
6918 case X86_XCPT_AC:
6919 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6920 /* no break! */
6921 default:
6922 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6923 break;
6924 }
6925 }
6926 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6927 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6928 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6929 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6930 else
6931 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6932
6933 rc = TRPMResetTrap(pVCpu);
6934 AssertRC(rc);
6935 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6936 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6937
6938 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6939}
6940
6941
6942/**
6943 * Converts the pending HM event into a TRPM trap.
6944 *
6945 * @param pVCpu The cross context virtual CPU structure.
6946 */
6947static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6948{
6949 Assert(pVCpu->hm.s.Event.fPending);
6950
6951 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6952 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6953 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6954 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6955
6956 /* If a trap was already pending, we did something wrong! */
6957 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6958
6959 TRPMEVENT enmTrapType;
6960 switch (uVectorType)
6961 {
6962 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6963 enmTrapType = TRPM_HARDWARE_INT;
6964 break;
6965
6966 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6967 enmTrapType = TRPM_SOFTWARE_INT;
6968 break;
6969
6970 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6971 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6972 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6973 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6974 enmTrapType = TRPM_TRAP;
6975 break;
6976
6977 default:
6978 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6979 enmTrapType = TRPM_32BIT_HACK;
6980 break;
6981 }
6982
6983 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6984
6985 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6986 AssertRC(rc);
6987
6988 if (fErrorCodeValid)
6989 TRPMSetErrorCode(pVCpu, uErrorCode);
6990
6991 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6992 && uVector == X86_XCPT_PF)
6993 {
6994 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6995 }
6996 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6997 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6998 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6999 {
7000 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7001 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7002 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7003 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7004 }
7005
7006 /* Clear any pending events from the VMCS. */
7007 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7008 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7009
7010 /* We're now done converting the pending event. */
7011 pVCpu->hm.s.Event.fPending = false;
7012}
7013
7014
7015/**
7016 * Does the necessary state syncing before returning to ring-3 for any reason
7017 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7018 *
7019 * @returns VBox status code.
7020 * @param pVM The cross context VM structure.
7021 * @param pVCpu The cross context virtual CPU structure.
7022 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7023 * be out-of-sync. Make sure to update the required
7024 * fields before using them.
7025 * @param fSaveGuestState Whether to save the guest state or not.
7026 *
7027 * @remarks No-long-jmp zone!!!
7028 */
7029static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7030{
7031 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7032 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7033
7034 RTCPUID idCpu = RTMpCpuId();
7035 Log4Func(("HostCpuId=%u\n", idCpu));
7036
7037 /*
7038 * !!! IMPORTANT !!!
7039 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7040 */
7041
7042 /* Save the guest state if necessary. */
7043 if ( fSaveGuestState
7044 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7045 {
7046 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7047 AssertRCReturn(rc, rc);
7048 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7049 }
7050
7051 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7052 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7053 {
7054 if (fSaveGuestState)
7055 {
7056 /* We shouldn't reload CR0 without saving it first. */
7057 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7058 AssertRCReturn(rc, rc);
7059 }
7060 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7061 }
7062
7063 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7064#ifdef VBOX_STRICT
7065 if (CPUMIsHyperDebugStateActive(pVCpu))
7066 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7067#endif
7068 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7069 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7070 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7071 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7072
7073#if HC_ARCH_BITS == 64
7074 /* Restore host-state bits that VT-x only restores partially. */
7075 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7076 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7077 {
7078 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7079 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7080 }
7081 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7082#endif
7083
7084 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7085 if (pVCpu->hm.s.vmx.fLazyMsrs)
7086 {
7087 /* We shouldn't reload the guest MSRs without saving it first. */
7088 if (!fSaveGuestState)
7089 {
7090 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7091 AssertRCReturn(rc, rc);
7092 }
7093 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7094 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7095 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7096 }
7097
7098 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7099 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7100
7101 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7102 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7103 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7104 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7105 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7106 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7107 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7108 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7109
7110 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7111
7112 /** @todo This partially defeats the purpose of having preemption hooks.
7113 * The problem is, deregistering the hooks should be moved to a place that
7114 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7115 * context.
7116 */
7117 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7118 {
7119 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7120 AssertRCReturn(rc, rc);
7121
7122 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7123 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7124 }
7125 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7126 NOREF(idCpu);
7127
7128 return VINF_SUCCESS;
7129}
7130
7131
7132/**
7133 * Leaves the VT-x session.
7134 *
7135 * @returns VBox status code.
7136 * @param pVM The cross context VM structure.
7137 * @param pVCpu The cross context virtual CPU structure.
7138 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7139 * out-of-sync. Make sure to update the required fields
7140 * before using them.
7141 *
7142 * @remarks No-long-jmp zone!!!
7143 */
7144DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7145{
7146 HM_DISABLE_PREEMPT();
7147 HMVMX_ASSERT_CPU_SAFE();
7148 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7149 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7150
7151 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7152 and done this from the VMXR0ThreadCtxCallback(). */
7153 if (!pVCpu->hm.s.fLeaveDone)
7154 {
7155 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7156 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7157 pVCpu->hm.s.fLeaveDone = true;
7158 }
7159 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7160
7161 /*
7162 * !!! IMPORTANT !!!
7163 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7164 */
7165
7166 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7167 /** @todo Deregistering here means we need to VMCLEAR always
7168 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7169 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7170 VMMR0ThreadCtxHookDisable(pVCpu);
7171
7172 /* Leave HM context. This takes care of local init (term). */
7173 int rc = HMR0LeaveCpu(pVCpu);
7174
7175 HM_RESTORE_PREEMPT();
7176 return rc;
7177}
7178
7179
7180/**
7181 * Does the necessary state syncing before doing a longjmp to ring-3.
7182 *
7183 * @returns VBox status code.
7184 * @param pVM The cross context VM structure.
7185 * @param pVCpu The cross context virtual CPU structure.
7186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7187 * out-of-sync. Make sure to update the required fields
7188 * before using them.
7189 *
7190 * @remarks No-long-jmp zone!!!
7191 */
7192DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7193{
7194 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7195}
7196
7197
7198/**
7199 * Take necessary actions before going back to ring-3.
7200 *
7201 * An action requires us to go back to ring-3. This function does the necessary
7202 * steps before we can safely return to ring-3. This is not the same as longjmps
7203 * to ring-3, this is voluntary and prepares the guest so it may continue
7204 * executing outside HM (recompiler/IEM).
7205 *
7206 * @returns VBox status code.
7207 * @param pVM The cross context VM structure.
7208 * @param pVCpu The cross context virtual CPU structure.
7209 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7210 * out-of-sync. Make sure to update the required fields
7211 * before using them.
7212 * @param rcExit The reason for exiting to ring-3. Can be
7213 * VINF_VMM_UNKNOWN_RING3_CALL.
7214 */
7215static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7216{
7217 Assert(pVM);
7218 Assert(pVCpu);
7219 Assert(pMixedCtx);
7220 HMVMX_ASSERT_PREEMPT_SAFE();
7221
7222 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7223 {
7224 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7225 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7226 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7227 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7228 }
7229
7230 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7231 VMMRZCallRing3Disable(pVCpu);
7232 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7233
7234 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7235 if (pVCpu->hm.s.Event.fPending)
7236 {
7237 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7238 Assert(!pVCpu->hm.s.Event.fPending);
7239 }
7240
7241 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7242 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7243
7244 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7245 and if we're injecting an event we should have a TRPM trap pending. */
7246 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7247#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7248 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7249#endif
7250
7251 /* Save guest state and restore host state bits. */
7252 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7253 AssertRCReturn(rc, rc);
7254 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7255 /* Thread-context hooks are unregistered at this point!!! */
7256
7257 /* Sync recompiler state. */
7258 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7259 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7260 | CPUM_CHANGED_LDTR
7261 | CPUM_CHANGED_GDTR
7262 | CPUM_CHANGED_IDTR
7263 | CPUM_CHANGED_TR
7264 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7265 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7266 if ( pVM->hm.s.fNestedPaging
7267 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7268 {
7269 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7270 }
7271
7272 Assert(!pVCpu->hm.s.fClearTrapFlag);
7273
7274 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7275 if (rcExit != VINF_EM_RAW_INTERRUPT)
7276 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7277
7278 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7279
7280 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7281 VMMRZCallRing3RemoveNotification(pVCpu);
7282 VMMRZCallRing3Enable(pVCpu);
7283
7284 return rc;
7285}
7286
7287
7288/**
7289 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7290 * longjump to ring-3 and possibly get preempted.
7291 *
7292 * @returns VBox status code.
7293 * @param pVCpu The cross context virtual CPU structure.
7294 * @param enmOperation The operation causing the ring-3 longjump.
7295 * @param pvUser Opaque pointer to the guest-CPU context. The data
7296 * may be out-of-sync. Make sure to update the required
7297 * fields before using them.
7298 */
7299static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7300{
7301 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7302 {
7303 /*
7304 * !!! IMPORTANT !!!
7305 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7306 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7307 */
7308 VMMRZCallRing3RemoveNotification(pVCpu);
7309 VMMRZCallRing3Disable(pVCpu);
7310 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7311 RTThreadPreemptDisable(&PreemptState);
7312
7313 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7314 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7315
7316#if HC_ARCH_BITS == 64
7317 /* Restore host-state bits that VT-x only restores partially. */
7318 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7319 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7320 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7321 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7322#endif
7323 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7324 if (pVCpu->hm.s.vmx.fLazyMsrs)
7325 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7326
7327 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7328 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7329 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7330 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7331 {
7332 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7333 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7334 }
7335
7336 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7337 VMMR0ThreadCtxHookDisable(pVCpu);
7338 HMR0LeaveCpu(pVCpu);
7339 RTThreadPreemptRestore(&PreemptState);
7340 return VINF_SUCCESS;
7341 }
7342
7343 Assert(pVCpu);
7344 Assert(pvUser);
7345 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7346 HMVMX_ASSERT_PREEMPT_SAFE();
7347
7348 VMMRZCallRing3Disable(pVCpu);
7349 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7350
7351 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7352 enmOperation));
7353
7354 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7355 AssertRCReturn(rc, rc);
7356
7357 VMMRZCallRing3Enable(pVCpu);
7358 return VINF_SUCCESS;
7359}
7360
7361
7362/**
7363 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7364 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7365 *
7366 * @param pVCpu The cross context virtual CPU structure.
7367 */
7368DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7369{
7370 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7371 {
7372 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7373 {
7374 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7375 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7376 AssertRC(rc);
7377 Log4(("Setup interrupt-window exiting\n"));
7378 }
7379 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7380}
7381
7382
7383/**
7384 * Clears the interrupt-window exiting control in the VMCS.
7385 *
7386 * @param pVCpu The cross context virtual CPU structure.
7387 */
7388DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7389{
7390 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7391 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7392 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7393 AssertRC(rc);
7394 Log4(("Cleared interrupt-window exiting\n"));
7395}
7396
7397
7398/**
7399 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7400 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7401 *
7402 * @param pVCpu The cross context virtual CPU structure.
7403 */
7404DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7405{
7406 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7407 {
7408 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7409 {
7410 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7411 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7412 AssertRC(rc);
7413 Log4(("Setup NMI-window exiting\n"));
7414 }
7415 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7416}
7417
7418
7419/**
7420 * Clears the NMI-window exiting control in the VMCS.
7421 *
7422 * @param pVCpu The cross context virtual CPU structure.
7423 */
7424DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7425{
7426 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7427 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7428 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7429 AssertRC(rc);
7430 Log4(("Cleared NMI-window exiting\n"));
7431}
7432
7433
7434/**
7435 * Evaluates the event to be delivered to the guest and sets it as the pending
7436 * event.
7437 *
7438 * @returns The VT-x guest-interruptibility state.
7439 * @param pVCpu The cross context virtual CPU structure.
7440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7441 * out-of-sync. Make sure to update the required fields
7442 * before using them.
7443 */
7444static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7445{
7446 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7447 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7448 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7449 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7450 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7451
7452 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7453 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7454 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7455 Assert(!TRPMHasTrap(pVCpu));
7456
7457#ifdef VBOX_WITH_NEW_APIC
7458 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7459 APICUpdatePendingInterrupts(pVCpu);
7460#endif
7461
7462 /*
7463 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7464 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7465 */
7466 /** @todo SMI. SMIs take priority over NMIs. */
7467 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7468 {
7469 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7470 if ( !pVCpu->hm.s.Event.fPending
7471 && !fBlockNmi
7472 && !fBlockSti
7473 && !fBlockMovSS)
7474 {
7475 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7476 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7477 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7478
7479 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7481 }
7482 else
7483 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7484 }
7485 /*
7486 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7487 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7488 */
7489 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7490 && !pVCpu->hm.s.fSingleInstruction)
7491 {
7492 Assert(!DBGFIsStepping(pVCpu));
7493 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7494 AssertRC(rc);
7495 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7496 if ( !pVCpu->hm.s.Event.fPending
7497 && !fBlockInt
7498 && !fBlockSti
7499 && !fBlockMovSS)
7500 {
7501 uint8_t u8Interrupt;
7502 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7503 if (RT_SUCCESS(rc))
7504 {
7505 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7506 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7507 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7508
7509 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7510 }
7511 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7512 {
7513 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7514 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7515 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7516 }
7517 else
7518 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7519 }
7520 else
7521 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7522 }
7523
7524 return uIntrState;
7525}
7526
7527
7528/**
7529 * Sets a pending-debug exception to be delivered to the guest if the guest is
7530 * single-stepping in the VMCS.
7531 *
7532 * @param pVCpu The cross context virtual CPU structure.
7533 */
7534DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7535{
7536 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7537 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7538 AssertRC(rc);
7539}
7540
7541
7542/**
7543 * Injects any pending events into the guest if the guest is in a state to
7544 * receive them.
7545 *
7546 * @returns Strict VBox status code (i.e. informational status codes too).
7547 * @param pVCpu The cross context virtual CPU structure.
7548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7549 * out-of-sync. Make sure to update the required fields
7550 * before using them.
7551 * @param uIntrState The VT-x guest-interruptibility state.
7552 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7553 * return VINF_EM_DBG_STEPPED if the event was
7554 * dispatched directly.
7555 */
7556static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7557{
7558 HMVMX_ASSERT_PREEMPT_SAFE();
7559 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7560
7561 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7562 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7563
7564 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7565 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7566 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7567 Assert(!TRPMHasTrap(pVCpu));
7568
7569 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7570 if (pVCpu->hm.s.Event.fPending)
7571 {
7572 /*
7573 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7574 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7575 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7576 *
7577 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7578 */
7579 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7580#ifdef VBOX_STRICT
7581 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7582 {
7583 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7584 Assert(!fBlockInt);
7585 Assert(!fBlockSti);
7586 Assert(!fBlockMovSS);
7587 }
7588 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7589 {
7590 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7591 Assert(!fBlockSti);
7592 Assert(!fBlockMovSS);
7593 Assert(!fBlockNmi);
7594 }
7595#endif
7596 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7597 (uint8_t)uIntType));
7598 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7599 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7600 fStepping, &uIntrState);
7601 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7602
7603 /* Update the interruptibility-state as it could have been changed by
7604 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7605 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7606 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7607
7608 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7609 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7610 else
7611 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7612 }
7613
7614 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7615 if ( fBlockSti
7616 || fBlockMovSS)
7617 {
7618 if (!pVCpu->hm.s.fSingleInstruction)
7619 {
7620 /*
7621 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7622 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7623 * See Intel spec. 27.3.4 "Saving Non-Register State".
7624 */
7625 Assert(!DBGFIsStepping(pVCpu));
7626 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7627 AssertRCReturn(rc2, rc2);
7628 if (pMixedCtx->eflags.Bits.u1TF)
7629 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7630 }
7631 else if (pMixedCtx->eflags.Bits.u1TF)
7632 {
7633 /*
7634 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7635 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7636 */
7637 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7638 uIntrState = 0;
7639 }
7640 }
7641
7642 /*
7643 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7644 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7645 */
7646 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7647 AssertRC(rc2);
7648
7649 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7650 NOREF(fBlockMovSS); NOREF(fBlockSti);
7651 return rcStrict;
7652}
7653
7654
7655/**
7656 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7657 *
7658 * @param pVCpu The cross context virtual CPU structure.
7659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7660 * out-of-sync. Make sure to update the required fields
7661 * before using them.
7662 */
7663DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7664{
7665 NOREF(pMixedCtx);
7666 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7667 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7668}
7669
7670
7671/**
7672 * Injects a double-fault (\#DF) exception into the VM.
7673 *
7674 * @returns Strict VBox status code (i.e. informational status codes too).
7675 * @param pVCpu The cross context virtual CPU structure.
7676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7677 * out-of-sync. Make sure to update the required fields
7678 * before using them.
7679 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7680 * and should return VINF_EM_DBG_STEPPED if the event
7681 * is injected directly (register modified by us, not
7682 * by hardware on VM-entry).
7683 * @param puIntrState Pointer to the current guest interruptibility-state.
7684 * This interruptibility-state will be updated if
7685 * necessary. This cannot not be NULL.
7686 */
7687DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7688{
7689 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7690 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7691 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7692 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7693 fStepping, puIntrState);
7694}
7695
7696
7697/**
7698 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7699 *
7700 * @param pVCpu The cross context virtual CPU structure.
7701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7702 * out-of-sync. Make sure to update the required fields
7703 * before using them.
7704 */
7705DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7706{
7707 NOREF(pMixedCtx);
7708 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7710 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7711}
7712
7713
7714/**
7715 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7716 *
7717 * @param pVCpu The cross context virtual CPU structure.
7718 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7719 * out-of-sync. Make sure to update the required fields
7720 * before using them.
7721 * @param cbInstr The value of RIP that is to be pushed on the guest
7722 * stack.
7723 */
7724DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7725{
7726 NOREF(pMixedCtx);
7727 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7728 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7729 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7730}
7731
7732
7733/**
7734 * Injects a general-protection (\#GP) fault into the VM.
7735 *
7736 * @returns Strict VBox status code (i.e. informational status codes too).
7737 * @param pVCpu The cross context virtual CPU structure.
7738 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7739 * out-of-sync. Make sure to update the required fields
7740 * before using them.
7741 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7742 * mode, i.e. in real-mode it's not valid).
7743 * @param u32ErrorCode The error code associated with the \#GP.
7744 * @param fStepping Whether we're running in
7745 * hmR0VmxRunGuestCodeStep() and should return
7746 * VINF_EM_DBG_STEPPED if the event is injected
7747 * directly (register modified by us, not by
7748 * hardware on VM-entry).
7749 * @param puIntrState Pointer to the current guest interruptibility-state.
7750 * This interruptibility-state will be updated if
7751 * necessary. This cannot not be NULL.
7752 */
7753DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7754 bool fStepping, uint32_t *puIntrState)
7755{
7756 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7757 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7758 if (fErrorCodeValid)
7759 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7760 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7761 fStepping, puIntrState);
7762}
7763
7764
7765/**
7766 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7767 * VM.
7768 *
7769 * @param pVCpu The cross context virtual CPU structure.
7770 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7771 * out-of-sync. Make sure to update the required fields
7772 * before using them.
7773 * @param u32ErrorCode The error code associated with the \#GP.
7774 */
7775DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7776{
7777 NOREF(pMixedCtx);
7778 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7779 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7780 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7781 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7782}
7783
7784
7785/**
7786 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7787 *
7788 * @param pVCpu The cross context virtual CPU structure.
7789 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7790 * out-of-sync. Make sure to update the required fields
7791 * before using them.
7792 * @param uVector The software interrupt vector number.
7793 * @param cbInstr The value of RIP that is to be pushed on the guest
7794 * stack.
7795 */
7796DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7797{
7798 NOREF(pMixedCtx);
7799 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7800 if ( uVector == X86_XCPT_BP
7801 || uVector == X86_XCPT_OF)
7802 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7803 else
7804 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7805 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7806}
7807
7808
7809/**
7810 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7811 * stack.
7812 *
7813 * @returns Strict VBox status code (i.e. informational status codes too).
7814 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7815 * @param pVM The cross context VM structure.
7816 * @param pMixedCtx Pointer to the guest-CPU context.
7817 * @param uValue The value to push to the guest stack.
7818 */
7819DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7820{
7821 /*
7822 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7823 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7824 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7825 */
7826 if (pMixedCtx->sp == 1)
7827 return VINF_EM_RESET;
7828 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7829 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7830 AssertRC(rc);
7831 return rc;
7832}
7833
7834
7835/**
7836 * Injects an event into the guest upon VM-entry by updating the relevant fields
7837 * in the VM-entry area in the VMCS.
7838 *
7839 * @returns Strict VBox status code (i.e. informational status codes too).
7840 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7841 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7842 *
7843 * @param pVCpu The cross context virtual CPU structure.
7844 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7845 * be out-of-sync. Make sure to update the required
7846 * fields before using them.
7847 * @param u64IntInfo The VM-entry interruption-information field.
7848 * @param cbInstr The VM-entry instruction length in bytes (for
7849 * software interrupts, exceptions and privileged
7850 * software exceptions).
7851 * @param u32ErrCode The VM-entry exception error code.
7852 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7853 * @param puIntrState Pointer to the current guest interruptibility-state.
7854 * This interruptibility-state will be updated if
7855 * necessary. This cannot not be NULL.
7856 * @param fStepping Whether we're running in
7857 * hmR0VmxRunGuestCodeStep() and should return
7858 * VINF_EM_DBG_STEPPED if the event is injected
7859 * directly (register modified by us, not by
7860 * hardware on VM-entry).
7861 *
7862 * @remarks Requires CR0!
7863 * @remarks No-long-jump zone!!!
7864 */
7865static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7866 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7867 uint32_t *puIntrState)
7868{
7869 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7870 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7871 Assert(puIntrState);
7872 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7873
7874 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7875 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7876
7877#ifdef VBOX_STRICT
7878 /* Validate the error-code-valid bit for hardware exceptions. */
7879 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7880 {
7881 switch (uVector)
7882 {
7883 case X86_XCPT_PF:
7884 case X86_XCPT_DF:
7885 case X86_XCPT_TS:
7886 case X86_XCPT_NP:
7887 case X86_XCPT_SS:
7888 case X86_XCPT_GP:
7889 case X86_XCPT_AC:
7890 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7891 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7892 /* fallthru */
7893 default:
7894 break;
7895 }
7896 }
7897#endif
7898
7899 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7900 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7901 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7902
7903 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7904
7905 /* We require CR0 to check if the guest is in real-mode. */
7906 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7907 AssertRCReturn(rc, rc);
7908
7909 /*
7910 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7911 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7912 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7913 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7914 */
7915 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7916 {
7917 PVM pVM = pVCpu->CTX_SUFF(pVM);
7918 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7919 {
7920 Assert(PDMVmmDevHeapIsEnabled(pVM));
7921 Assert(pVM->hm.s.vmx.pRealModeTSS);
7922
7923 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7924 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7925 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7926 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7927 AssertRCReturn(rc, rc);
7928 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7929
7930 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7931 size_t const cbIdtEntry = sizeof(X86IDTR16);
7932 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7933 {
7934 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7935 if (uVector == X86_XCPT_DF)
7936 return VINF_EM_RESET;
7937
7938 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7939 if (uVector == X86_XCPT_GP)
7940 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7941
7942 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7943 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7944 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7945 fStepping, puIntrState);
7946 }
7947
7948 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7949 uint16_t uGuestIp = pMixedCtx->ip;
7950 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7951 {
7952 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7953 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7954 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7955 }
7956 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7957 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7958
7959 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7960 X86IDTR16 IdtEntry;
7961 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7962 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7963 AssertRCReturn(rc, rc);
7964
7965 /* Construct the stack frame for the interrupt/exception handler. */
7966 VBOXSTRICTRC rcStrict;
7967 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7968 if (rcStrict == VINF_SUCCESS)
7969 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7970 if (rcStrict == VINF_SUCCESS)
7971 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7972
7973 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7974 if (rcStrict == VINF_SUCCESS)
7975 {
7976 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7977 pMixedCtx->rip = IdtEntry.offSel;
7978 pMixedCtx->cs.Sel = IdtEntry.uSel;
7979 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7980 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7981 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7982 && uVector == X86_XCPT_PF)
7983 pMixedCtx->cr2 = GCPtrFaultAddress;
7984
7985 /* If any other guest-state bits are changed here, make sure to update
7986 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7987 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7988 | HM_CHANGED_GUEST_RIP
7989 | HM_CHANGED_GUEST_RFLAGS
7990 | HM_CHANGED_GUEST_RSP);
7991
7992 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7993 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7994 {
7995 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7996 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7997 Log4(("Clearing inhibition due to STI.\n"));
7998 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7999 }
8000 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8001 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8002
8003 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8004 it, if we are returning to ring-3 before executing guest code. */
8005 pVCpu->hm.s.Event.fPending = false;
8006
8007 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8008 if (fStepping)
8009 rcStrict = VINF_EM_DBG_STEPPED;
8010 }
8011 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8012 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8013 return rcStrict;
8014 }
8015
8016 /*
8017 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8018 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8019 */
8020 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8021 }
8022
8023 /* Validate. */
8024 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8025 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8026 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8027
8028 /* Inject. */
8029 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8030 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8031 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8032 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8033
8034 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8035 && uVector == X86_XCPT_PF)
8036 pMixedCtx->cr2 = GCPtrFaultAddress;
8037
8038 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8039 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8040
8041 AssertRCReturn(rc, rc);
8042 return VINF_SUCCESS;
8043}
8044
8045
8046/**
8047 * Clears the interrupt-window exiting control in the VMCS and if necessary
8048 * clears the current event in the VMCS as well.
8049 *
8050 * @returns VBox status code.
8051 * @param pVCpu The cross context virtual CPU structure.
8052 *
8053 * @remarks Use this function only to clear events that have not yet been
8054 * delivered to the guest but are injected in the VMCS!
8055 * @remarks No-long-jump zone!!!
8056 */
8057static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8058{
8059 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8060
8061 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8062 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8063
8064 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8065 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8066}
8067
8068
8069/**
8070 * Enters the VT-x session.
8071 *
8072 * @returns VBox status code.
8073 * @param pVM The cross context VM structure.
8074 * @param pVCpu The cross context virtual CPU structure.
8075 * @param pCpu Pointer to the CPU info struct.
8076 */
8077VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8078{
8079 AssertPtr(pVM);
8080 AssertPtr(pVCpu);
8081 Assert(pVM->hm.s.vmx.fSupported);
8082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8083 NOREF(pCpu); NOREF(pVM);
8084
8085 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8086 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8087
8088#ifdef VBOX_STRICT
8089 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8090 RTCCUINTREG uHostCR4 = ASMGetCR4();
8091 if (!(uHostCR4 & X86_CR4_VMXE))
8092 {
8093 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8094 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8095 }
8096#endif
8097
8098 /*
8099 * Load the VCPU's VMCS as the current (and active) one.
8100 */
8101 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8102 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8103 if (RT_FAILURE(rc))
8104 return rc;
8105
8106 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8107 pVCpu->hm.s.fLeaveDone = false;
8108 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8109
8110 return VINF_SUCCESS;
8111}
8112
8113
8114/**
8115 * The thread-context callback (only on platforms which support it).
8116 *
8117 * @param enmEvent The thread-context event.
8118 * @param pVCpu The cross context virtual CPU structure.
8119 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8120 * @thread EMT(pVCpu)
8121 */
8122VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8123{
8124 NOREF(fGlobalInit);
8125
8126 switch (enmEvent)
8127 {
8128 case RTTHREADCTXEVENT_OUT:
8129 {
8130 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8131 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8132 VMCPU_ASSERT_EMT(pVCpu);
8133
8134 PVM pVM = pVCpu->CTX_SUFF(pVM);
8135 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8136
8137 /* No longjmps (logger flushes, locks) in this fragile context. */
8138 VMMRZCallRing3Disable(pVCpu);
8139 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8140
8141 /*
8142 * Restore host-state (FPU, debug etc.)
8143 */
8144 if (!pVCpu->hm.s.fLeaveDone)
8145 {
8146 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8147 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8148 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8149 pVCpu->hm.s.fLeaveDone = true;
8150 }
8151
8152 /* Leave HM context, takes care of local init (term). */
8153 int rc = HMR0LeaveCpu(pVCpu);
8154 AssertRC(rc); NOREF(rc);
8155
8156 /* Restore longjmp state. */
8157 VMMRZCallRing3Enable(pVCpu);
8158 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8159 break;
8160 }
8161
8162 case RTTHREADCTXEVENT_IN:
8163 {
8164 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8165 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8166 VMCPU_ASSERT_EMT(pVCpu);
8167
8168 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8169 VMMRZCallRing3Disable(pVCpu);
8170 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8171
8172 /* Initialize the bare minimum state required for HM. This takes care of
8173 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8174 int rc = HMR0EnterCpu(pVCpu);
8175 AssertRC(rc);
8176 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8177
8178 /* Load the active VMCS as the current one. */
8179 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8180 {
8181 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8182 AssertRC(rc); NOREF(rc);
8183 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8184 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8185 }
8186 pVCpu->hm.s.fLeaveDone = false;
8187
8188 /* Restore longjmp state. */
8189 VMMRZCallRing3Enable(pVCpu);
8190 break;
8191 }
8192
8193 default:
8194 break;
8195 }
8196}
8197
8198
8199/**
8200 * Saves the host state in the VMCS host-state.
8201 * Sets up the VM-exit MSR-load area.
8202 *
8203 * The CPU state will be loaded from these fields on every successful VM-exit.
8204 *
8205 * @returns VBox status code.
8206 * @param pVM The cross context VM structure.
8207 * @param pVCpu The cross context virtual CPU structure.
8208 *
8209 * @remarks No-long-jump zone!!!
8210 */
8211static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8212{
8213 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8214
8215 int rc = VINF_SUCCESS;
8216 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8217 {
8218 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8219 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8220
8221 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8222 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8223
8224 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8225 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8226
8227 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8228 }
8229 return rc;
8230}
8231
8232
8233/**
8234 * Saves the host state in the VMCS host-state.
8235 *
8236 * @returns VBox status code.
8237 * @param pVM The cross context VM structure.
8238 * @param pVCpu The cross context virtual CPU structure.
8239 *
8240 * @remarks No-long-jump zone!!!
8241 */
8242VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8243{
8244 AssertPtr(pVM);
8245 AssertPtr(pVCpu);
8246
8247 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8248
8249 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8250 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8251 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8252 return hmR0VmxSaveHostState(pVM, pVCpu);
8253}
8254
8255
8256/**
8257 * Loads the guest state into the VMCS guest-state area.
8258 *
8259 * The will typically be done before VM-entry when the guest-CPU state and the
8260 * VMCS state may potentially be out of sync.
8261 *
8262 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8263 * VM-entry controls.
8264 * Sets up the appropriate VMX non-root function to execute guest code based on
8265 * the guest CPU mode.
8266 *
8267 * @returns VBox strict status code.
8268 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8269 * without unrestricted guest access and the VMMDev is not presently
8270 * mapped (e.g. EFI32).
8271 *
8272 * @param pVM The cross context VM structure.
8273 * @param pVCpu The cross context virtual CPU structure.
8274 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8275 * out-of-sync. Make sure to update the required fields
8276 * before using them.
8277 *
8278 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8279 * caller disables then again on successfull return. Confusing.)
8280 */
8281static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8282{
8283 AssertPtr(pVM);
8284 AssertPtr(pVCpu);
8285 AssertPtr(pMixedCtx);
8286 HMVMX_ASSERT_PREEMPT_SAFE();
8287
8288 VMMRZCallRing3Disable(pVCpu);
8289 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8290
8291 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8292
8293 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8294
8295 /* Determine real-on-v86 mode. */
8296 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8297 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8298 && CPUMIsGuestInRealModeEx(pMixedCtx))
8299 {
8300 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8301 }
8302
8303 /*
8304 * Load the guest-state into the VMCS.
8305 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8306 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8307 */
8308 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8309 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8310
8311 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8312 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8313 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8314
8315 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8316 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8317 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8318
8319 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8320 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8321
8322 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8323 if (rcStrict == VINF_SUCCESS)
8324 { /* likely */ }
8325 else
8326 {
8327 VMMRZCallRing3Enable(pVCpu);
8328 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8329 return rcStrict;
8330 }
8331
8332 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8333 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8334 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8335
8336 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8337 determine we don't have to swap EFER after all. */
8338 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8339 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8340
8341 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8342 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8343
8344 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8345 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8346
8347 /*
8348 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8349 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8350 */
8351 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8352 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8353
8354 /* Clear any unused and reserved bits. */
8355 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8356
8357 VMMRZCallRing3Enable(pVCpu);
8358
8359 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8360 return rc;
8361}
8362
8363
8364/**
8365 * Loads the state shared between the host and guest into the VMCS.
8366 *
8367 * @param pVM The cross context VM structure.
8368 * @param pVCpu The cross context virtual CPU structure.
8369 * @param pCtx Pointer to the guest-CPU context.
8370 *
8371 * @remarks No-long-jump zone!!!
8372 */
8373static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8374{
8375 NOREF(pVM);
8376
8377 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8378 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8379
8380 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8381 {
8382 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8383 AssertRC(rc);
8384 }
8385
8386 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8387 {
8388 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8389 AssertRC(rc);
8390
8391 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8392 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8393 {
8394 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8395 AssertRC(rc);
8396 }
8397 }
8398
8399 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8400 {
8401 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8402 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8403 }
8404
8405 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8406 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8407 {
8408 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8409 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8410 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8411 AssertRC(rc);
8412 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8413 }
8414
8415 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8416 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8417}
8418
8419
8420/**
8421 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8422 *
8423 * @returns Strict VBox status code (i.e. informational status codes too).
8424 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8425 * without unrestricted guest access and the VMMDev is not presently
8426 * mapped (e.g. EFI32).
8427 *
8428 * @param pVM The cross context VM structure.
8429 * @param pVCpu The cross context virtual CPU structure.
8430 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8431 * out-of-sync. Make sure to update the required fields
8432 * before using them.
8433 */
8434static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8435{
8436 HMVMX_ASSERT_PREEMPT_SAFE();
8437
8438 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8439#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8440 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8441#endif
8442
8443 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8444 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8445 {
8446 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8447 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8448 { /* likely */}
8449 else
8450 {
8451 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8452 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8453 }
8454 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8455 }
8456 else if (HMCPU_CF_VALUE(pVCpu))
8457 {
8458 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8459 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8460 { /* likely */}
8461 else
8462 {
8463 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8464 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8465 return rcStrict;
8466 }
8467 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8468 }
8469
8470 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8471 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8472 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8473 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8474 return rcStrict;
8475}
8476
8477
8478/**
8479 * Does the preparations before executing guest code in VT-x.
8480 *
8481 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8482 * recompiler/IEM. We must be cautious what we do here regarding committing
8483 * guest-state information into the VMCS assuming we assuredly execute the
8484 * guest in VT-x mode.
8485 *
8486 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8487 * the common-state (TRPM/forceflags), we must undo those changes so that the
8488 * recompiler/IEM can (and should) use them when it resumes guest execution.
8489 * Otherwise such operations must be done when we can no longer exit to ring-3.
8490 *
8491 * @returns Strict VBox status code (i.e. informational status codes too).
8492 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8493 * have been disabled.
8494 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8495 * double-fault into the guest.
8496 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8497 * dispatched directly.
8498 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8499 *
8500 * @param pVM The cross context VM structure.
8501 * @param pVCpu The cross context virtual CPU structure.
8502 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8503 * out-of-sync. Make sure to update the required fields
8504 * before using them.
8505 * @param pVmxTransient Pointer to the VMX transient structure.
8506 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8507 * us ignore some of the reasons for returning to
8508 * ring-3, and return VINF_EM_DBG_STEPPED if event
8509 * dispatching took place.
8510 */
8511static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8512{
8513 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8514
8515#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8516 PGMRZDynMapFlushAutoSet(pVCpu);
8517#endif
8518
8519 /* Check force flag actions that might require us to go back to ring-3. */
8520 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8521 if (rcStrict == VINF_SUCCESS)
8522 { /* FFs doesn't get set all the time. */ }
8523 else
8524 return rcStrict;
8525
8526#ifndef IEM_VERIFICATION_MODE_FULL
8527 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8528 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8529 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8530 {
8531 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8532 RTGCPHYS GCPhysApicBase;
8533 GCPhysApicBase = pMixedCtx->msrApicBase;
8534 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8535
8536 /* Unalias any existing mapping. */
8537 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8538 AssertRCReturn(rc, rc);
8539
8540 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8541 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8542 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8543 AssertRCReturn(rc, rc);
8544
8545 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8546 }
8547#endif /* !IEM_VERIFICATION_MODE_FULL */
8548
8549 if (TRPMHasTrap(pVCpu))
8550 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8551 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8552
8553 /*
8554 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8555 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8556 */
8557 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8558 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8559 { /* likely */ }
8560 else
8561 {
8562 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8563 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8564 return rcStrict;
8565 }
8566
8567 /*
8568 * Load the guest state bits, we can handle longjmps/getting preempted here.
8569 *
8570 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8571 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8572 * Hence, this needs to be done -after- injection of events.
8573 */
8574 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8575 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8576 { /* likely */ }
8577 else
8578 return rcStrict;
8579
8580 /*
8581 * No longjmps to ring-3 from this point on!!!
8582 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8583 * This also disables flushing of the R0-logger instance (if any).
8584 */
8585 VMMRZCallRing3Disable(pVCpu);
8586
8587 /*
8588 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8589 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8590 *
8591 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8592 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8593 *
8594 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8595 * executing guest code.
8596 */
8597 pVmxTransient->fEFlags = ASMIntDisableFlags();
8598
8599 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8600 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8601 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8602 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8603 {
8604 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8605 {
8606 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8607 pVCpu->hm.s.Event.fPending = false;
8608
8609 return VINF_SUCCESS;
8610 }
8611
8612 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8613 rcStrict = VINF_EM_RAW_INTERRUPT;
8614 }
8615 else
8616 {
8617 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8618 rcStrict = VINF_EM_RAW_TO_R3;
8619 }
8620
8621 ASMSetFlags(pVmxTransient->fEFlags);
8622 VMMRZCallRing3Enable(pVCpu);
8623
8624 return rcStrict;
8625}
8626
8627
8628/**
8629 * Prepares to run guest code in VT-x and we've committed to doing so. This
8630 * means there is no backing out to ring-3 or anywhere else at this
8631 * point.
8632 *
8633 * @param pVM The cross context VM structure.
8634 * @param pVCpu The cross context virtual CPU structure.
8635 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8636 * out-of-sync. Make sure to update the required fields
8637 * before using them.
8638 * @param pVmxTransient Pointer to the VMX transient structure.
8639 *
8640 * @remarks Called with preemption disabled.
8641 * @remarks No-long-jump zone!!!
8642 */
8643static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8644{
8645 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8646 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8647 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8648
8649 /*
8650 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8651 */
8652 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8653 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8654
8655#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8656 if (!CPUMIsGuestFPUStateActive(pVCpu))
8657 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8658 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8659 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8660#endif
8661
8662 if ( pVCpu->hm.s.fPreloadGuestFpu
8663 && !CPUMIsGuestFPUStateActive(pVCpu))
8664 {
8665 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8666 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8667 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8668 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8669 }
8670
8671 /*
8672 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8673 */
8674 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8675 && pVCpu->hm.s.vmx.cMsrs > 0)
8676 {
8677 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8678 }
8679
8680 /*
8681 * Load the host state bits as we may've been preempted (only happens when
8682 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8683 */
8684 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8685 * any effect to the host state needing to be saved? */
8686 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8687 {
8688 /* This ASSUMES that pfnStartVM has been set up already. */
8689 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8690 AssertRC(rc);
8691 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8692 }
8693 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8694
8695 /*
8696 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8697 */
8698 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8699 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8700 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8701
8702 /* Store status of the shared guest-host state at the time of VM-entry. */
8703#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8704 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8705 {
8706 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8707 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8708 }
8709 else
8710#endif
8711 {
8712 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8713 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8714 }
8715 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8716
8717 /*
8718 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8719 */
8720 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8721 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8722
8723 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8724 RTCPUID idCurrentCpu = pCpu->idCpu;
8725 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8726 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8727 {
8728 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8729 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8730 }
8731
8732 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8733 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8734 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8735 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8736
8737 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8738
8739 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8740 to start executing. */
8741
8742 /*
8743 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8744 */
8745 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8746 {
8747 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8748 {
8749 bool fMsrUpdated;
8750 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8751 AssertRC(rc2);
8752 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8753
8754 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8755 &fMsrUpdated);
8756 AssertRC(rc2);
8757 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8758
8759 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8760 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8761 }
8762 else
8763 {
8764 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8765 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8766 }
8767 }
8768
8769#ifdef VBOX_STRICT
8770 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8771 hmR0VmxCheckHostEferMsr(pVCpu);
8772 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8773#endif
8774#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8775 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8776 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8777 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8778#endif
8779}
8780
8781
8782/**
8783 * Performs some essential restoration of state after running guest code in
8784 * VT-x.
8785 *
8786 * @param pVM The cross context VM structure.
8787 * @param pVCpu The cross context virtual CPU structure.
8788 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8789 * out-of-sync. Make sure to update the required fields
8790 * before using them.
8791 * @param pVmxTransient Pointer to the VMX transient structure.
8792 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8793 *
8794 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8795 *
8796 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8797 * unconditionally when it is safe to do so.
8798 */
8799static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8800{
8801 NOREF(pVM);
8802
8803 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8804
8805 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8806 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8807 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8808 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8809 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8810 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8811
8812 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8813 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8814
8815 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8816 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8817 Assert(!ASMIntAreEnabled());
8818 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8819
8820#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8821 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8822 {
8823 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8824 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8825 }
8826#endif
8827
8828#if HC_ARCH_BITS == 64
8829 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8830#endif
8831 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8832#ifdef VBOX_STRICT
8833 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8834#endif
8835 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8836 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8837
8838 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8839 uint32_t uExitReason;
8840 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8841 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8842 AssertRC(rc);
8843 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8844 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8845
8846 /* Update the VM-exit history array. */
8847 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8848
8849 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8850 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8851 {
8852 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8853 pVmxTransient->fVMEntryFailed));
8854 return;
8855 }
8856
8857 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8858 {
8859 /** @todo We can optimize this by only syncing with our force-flags when
8860 * really needed and keeping the VMCS state as it is for most
8861 * VM-exits. */
8862 /* Update the guest interruptibility-state from the VMCS. */
8863 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8864
8865#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8866 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8867 AssertRC(rc);
8868#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8869 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8870 AssertRC(rc);
8871#endif
8872
8873 /*
8874 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8875 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8876 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8877 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8878 */
8879 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8880 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8881 {
8882 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8883 AssertRC(rc);
8884 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8885 }
8886 }
8887}
8888
8889
8890/**
8891 * Runs the guest code using VT-x the normal way.
8892 *
8893 * @returns VBox status code.
8894 * @param pVM The cross context VM structure.
8895 * @param pVCpu The cross context virtual CPU structure.
8896 * @param pCtx Pointer to the guest-CPU context.
8897 *
8898 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8899 */
8900static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8901{
8902 VMXTRANSIENT VmxTransient;
8903 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8904 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8905 uint32_t cLoops = 0;
8906
8907 for (;; cLoops++)
8908 {
8909 Assert(!HMR0SuspendPending());
8910 HMVMX_ASSERT_CPU_SAFE();
8911
8912 /* Preparatory work for running guest code, this may force us to return
8913 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8914 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8915 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8916 if (rcStrict != VINF_SUCCESS)
8917 break;
8918
8919 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8920 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8921 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8922
8923 /* Restore any residual host-state and save any bits shared between host
8924 and guest into the guest-CPU state. Re-enables interrupts! */
8925 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8926
8927 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8928 if (RT_SUCCESS(rcRun))
8929 { /* very likely */ }
8930 else
8931 {
8932 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8933 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8934 return rcRun;
8935 }
8936
8937 /* Profile the VM-exit. */
8938 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8939 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8940 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8941 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8942 HMVMX_START_EXIT_DISPATCH_PROF();
8943
8944 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8945
8946 /* Handle the VM-exit. */
8947#ifdef HMVMX_USE_FUNCTION_TABLE
8948 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8949#else
8950 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8951#endif
8952 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8953 if (rcStrict == VINF_SUCCESS)
8954 {
8955 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8956 continue; /* likely */
8957 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8958 rcStrict = VINF_EM_RAW_INTERRUPT;
8959 }
8960 break;
8961 }
8962
8963 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8964 return rcStrict;
8965}
8966
8967
8968
8969/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8970 * probes.
8971 *
8972 * The following few functions and associated structure contains the bloat
8973 * necessary for providing detailed debug events and dtrace probes as well as
8974 * reliable host side single stepping. This works on the principle of
8975 * "subclassing" the normal execution loop and workers. We replace the loop
8976 * method completely and override selected helpers to add necessary adjustments
8977 * to their core operation.
8978 *
8979 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8980 * any performance for debug and analysis features.
8981 *
8982 * @{
8983 */
8984
8985typedef struct VMXRUNDBGSTATE
8986{
8987 /** The RIP we started executing at. This is for detecting that we stepped. */
8988 uint64_t uRipStart;
8989 /** The CS we started executing with. */
8990 uint16_t uCsStart;
8991
8992 /** Whether we've actually modified the 1st execution control field. */
8993 bool fModifiedProcCtls : 1;
8994 /** Whether we've actually modified the 2nd execution control field. */
8995 bool fModifiedProcCtls2 : 1;
8996 /** Whether we've actually modified the exception bitmap. */
8997 bool fModifiedXcptBitmap : 1;
8998
8999 /** We desire the modified the CR0 mask to be cleared. */
9000 bool fClearCr0Mask : 1;
9001 /** We desire the modified the CR4 mask to be cleared. */
9002 bool fClearCr4Mask : 1;
9003 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9004 uint32_t fCpe1Extra;
9005 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9006 uint32_t fCpe1Unwanted;
9007 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9008 uint32_t fCpe2Extra;
9009 /** Extra stuff we need in */
9010 uint32_t bmXcptExtra;
9011 /** The sequence number of the Dtrace provider settings the state was
9012 * configured against. */
9013 uint32_t uDtraceSettingsSeqNo;
9014 /** Exits to check (one bit per exit). */
9015 uint32_t bmExitsToCheck[3];
9016
9017 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9018 uint32_t fProcCtlsInitial;
9019 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9020 uint32_t fProcCtls2Initial;
9021 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9022 uint32_t bmXcptInitial;
9023} VMXRUNDBGSTATE;
9024AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9025typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9026
9027
9028/**
9029 * Initializes the VMXRUNDBGSTATE structure.
9030 *
9031 * @param pVCpu The cross context virtual CPU structure of the
9032 * calling EMT.
9033 * @param pCtx The CPU register context to go with @a pVCpu.
9034 * @param pDbgState The structure to initialize.
9035 */
9036DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9037{
9038 pDbgState->uRipStart = pCtx->rip;
9039 pDbgState->uCsStart = pCtx->cs.Sel;
9040
9041 pDbgState->fModifiedProcCtls = false;
9042 pDbgState->fModifiedProcCtls2 = false;
9043 pDbgState->fModifiedXcptBitmap = false;
9044 pDbgState->fClearCr0Mask = false;
9045 pDbgState->fClearCr4Mask = false;
9046 pDbgState->fCpe1Extra = 0;
9047 pDbgState->fCpe1Unwanted = 0;
9048 pDbgState->fCpe2Extra = 0;
9049 pDbgState->bmXcptExtra = 0;
9050 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9051 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9052 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9053}
9054
9055
9056/**
9057 * Updates the VMSC fields with changes requested by @a pDbgState.
9058 *
9059 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9060 * immediately before executing guest code, i.e. when interrupts are disabled.
9061 * We don't check status codes here as we cannot easily assert or return in the
9062 * latter case.
9063 *
9064 * @param pVCpu The cross context virtual CPU structure.
9065 * @param pDbgState The debug state.
9066 */
9067DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9068{
9069 /*
9070 * Ensure desired flags in VMCS control fields are set.
9071 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9072 *
9073 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9074 * there should be no stale data in pCtx at this point.
9075 */
9076 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9077 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9078 {
9079 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9080 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9081 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9082 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9083 pDbgState->fModifiedProcCtls = true;
9084 }
9085
9086 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9087 {
9088 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9089 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9090 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9091 pDbgState->fModifiedProcCtls2 = true;
9092 }
9093
9094 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9095 {
9096 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9097 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9098 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9099 pDbgState->fModifiedXcptBitmap = true;
9100 }
9101
9102 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9103 {
9104 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9105 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9106 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9107 }
9108
9109 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9110 {
9111 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9112 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9113 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9114 }
9115}
9116
9117
9118DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9119{
9120 /*
9121 * Restore exit control settings as we may not reenter this function the
9122 * next time around.
9123 */
9124 /* We reload the initial value, trigger what we can of recalculations the
9125 next time around. From the looks of things, that's all that's required atm. */
9126 if (pDbgState->fModifiedProcCtls)
9127 {
9128 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9129 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9130 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9131 AssertRCReturn(rc2, rc2);
9132 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9133 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9134 }
9135
9136 /* We're currently the only ones messing with this one, so just restore the
9137 cached value and reload the field. */
9138 if ( pDbgState->fModifiedProcCtls2
9139 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9140 {
9141 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9142 AssertRCReturn(rc2, rc2);
9143 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9144 }
9145
9146 /* If we've modified the exception bitmap, we restore it and trigger
9147 reloading and partial recalculation the next time around. */
9148 if (pDbgState->fModifiedXcptBitmap)
9149 {
9150 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9151 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9152 }
9153
9154 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9155 if (pDbgState->fClearCr0Mask)
9156 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9157
9158 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9159 if (pDbgState->fClearCr4Mask)
9160 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9161
9162 return rcStrict;
9163}
9164
9165
9166/**
9167 * Configures VM-exit controls for current DBGF and DTrace settings.
9168 *
9169 * This updates @a pDbgState and the VMCS execution control fields to reflect
9170 * the necessary exits demanded by DBGF and DTrace.
9171 *
9172 * @param pVM The cross context VM structure.
9173 * @param pVCpu The cross context virtual CPU structure.
9174 * @param pCtx Pointer to the guest-CPU context.
9175 * @param pDbgState The debug state.
9176 * @param pVmxTransient Pointer to the VMX transient structure. May update
9177 * fUpdateTscOffsettingAndPreemptTimer.
9178 */
9179static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9180 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9181{
9182 /*
9183 * Take down the dtrace serial number so we can spot changes.
9184 */
9185 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9186 ASMCompilerBarrier();
9187
9188 /*
9189 * We'll rebuild most of the middle block of data members (holding the
9190 * current settings) as we go along here, so start by clearing it all.
9191 */
9192 pDbgState->bmXcptExtra = 0;
9193 pDbgState->fCpe1Extra = 0;
9194 pDbgState->fCpe1Unwanted = 0;
9195 pDbgState->fCpe2Extra = 0;
9196 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9197 pDbgState->bmExitsToCheck[i] = 0;
9198
9199 /*
9200 * Software interrupts (INT XXh) - no idea how to trigger these...
9201 */
9202 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9203 || VBOXVMM_INT_SOFTWARE_ENABLED())
9204 {
9205 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9206 }
9207
9208 /*
9209 * Exception bitmap and XCPT events+probes.
9210 */
9211 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9212 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9213 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9214
9215 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9216 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9217 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9218 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9219 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9220 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9221 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9222 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9223 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9224 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9225 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9226 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9227 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9228 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9229 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9230 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9231 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9232 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9233
9234 if (pDbgState->bmXcptExtra)
9235 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9236
9237 /*
9238 * Process events and probes for VM exits, making sure we get the wanted exits.
9239 *
9240 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9241 * So, when adding/changing/removing please don't forget to update it.
9242 *
9243 * Some of the macros are picking up local variables to save horizontal space,
9244 * (being able to see it in a table is the lesser evil here).
9245 */
9246#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9247 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9248 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9249#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9250 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9251 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9252 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9253 } else do { } while (0)
9254#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9255 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9256 { \
9257 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9258 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9259 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9260 } else do { } while (0)
9261#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9262 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9263 { \
9264 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9265 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9266 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9267 } else do { } while (0)
9268#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9269 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9270 { \
9271 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9272 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9273 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9274 } else do { } while (0)
9275
9276 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9277 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9278 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9279 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9280 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9281
9282 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9284 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9286 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9288 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9290 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9292 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9294 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9308 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9309 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9310 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9311 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9312 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9313 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9314 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9315 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9316 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9317 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9318
9319 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9320 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9321 {
9322 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9323 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9324 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9325 AssertRC(rc2);
9326
9327#if 0 /** @todo fix me */
9328 pDbgState->fClearCr0Mask = true;
9329 pDbgState->fClearCr4Mask = true;
9330#endif
9331 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9332 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9333 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9334 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9335 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9336 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9337 require clearing here and in the loop if we start using it. */
9338 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9339 }
9340 else
9341 {
9342 if (pDbgState->fClearCr0Mask)
9343 {
9344 pDbgState->fClearCr0Mask = false;
9345 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9346 }
9347 if (pDbgState->fClearCr4Mask)
9348 {
9349 pDbgState->fClearCr4Mask = false;
9350 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9351 }
9352 }
9353 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9354 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9355
9356 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9357 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9358 {
9359 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9360 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9361 }
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9364
9365 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9366 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9367 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9368 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9369 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9370 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9371 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9372 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9373#if 0 /** @todo too slow, fix handler. */
9374 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9375#endif
9376 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9377
9378 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9379 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9380 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9381 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9382 {
9383 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9384 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9385 }
9386 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9387 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9388 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9389 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9390
9391 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9392 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9393 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9394 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9395 {
9396 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9397 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9398 }
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9400 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9402 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9403
9404 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9406 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9408 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9410 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9414 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9416 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9417 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9418 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9420 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9421 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9422 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9423 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9424 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9425 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9426
9427#undef IS_EITHER_ENABLED
9428#undef SET_ONLY_XBM_IF_EITHER_EN
9429#undef SET_CPE1_XBM_IF_EITHER_EN
9430#undef SET_CPEU_XBM_IF_EITHER_EN
9431#undef SET_CPE2_XBM_IF_EITHER_EN
9432
9433 /*
9434 * Sanitize the control stuff.
9435 */
9436 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9437 if (pDbgState->fCpe2Extra)
9438 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9439 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9440 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9441 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9442 {
9443 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9444 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9445 }
9446
9447 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9448 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9449 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9450 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9451}
9452
9453
9454/**
9455 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9456 *
9457 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9458 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9459 * either.
9460 *
9461 * @returns Strict VBox status code (i.e. informational status codes too).
9462 * @param pVM The cross context VM structure.
9463 * @param pVCpu The cross context virtual CPU structure.
9464 * @param pMixedCtx Pointer to the guest-CPU context.
9465 * @param pVmxTransient Pointer to the VMX-transient structure.
9466 * @param uExitReason The VM-exit reason.
9467 *
9468 * @remarks The name of this function is displayed by dtrace, so keep it short
9469 * and to the point. No longer than 33 chars long, please.
9470 */
9471static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9472 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9473{
9474 /*
9475 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9476 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9477 *
9478 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9479 * does. Must add/change/remove both places. Same ordering, please.
9480 *
9481 * Added/removed events must also be reflected in the next section
9482 * where we dispatch dtrace events.
9483 */
9484 bool fDtrace1 = false;
9485 bool fDtrace2 = false;
9486 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9487 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9488 uint32_t uEventArg = 0;
9489#define SET_EXIT(a_EventSubName) \
9490 do { \
9491 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9492 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9493 } while (0)
9494#define SET_BOTH(a_EventSubName) \
9495 do { \
9496 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9497 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9498 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9499 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9500 } while (0)
9501 switch (uExitReason)
9502 {
9503 case VMX_EXIT_MTF:
9504 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9505
9506 case VMX_EXIT_XCPT_OR_NMI:
9507 {
9508 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9509 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9510 {
9511 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9512 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9513 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9514 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9515 {
9516 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9517 {
9518 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9519 uEventArg = pVmxTransient->uExitIntErrorCode;
9520 }
9521 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9522 switch (enmEvent1)
9523 {
9524 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9525 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9526 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9527 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9528 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9529 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9530 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9531 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9532 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9533 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9534 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9535 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9536 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9537 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9538 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9539 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9540 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9541 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9542 default: break;
9543 }
9544 }
9545 else
9546 AssertFailed();
9547 break;
9548
9549 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9550 uEventArg = idxVector;
9551 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9552 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9553 break;
9554 }
9555 break;
9556 }
9557
9558 case VMX_EXIT_TRIPLE_FAULT:
9559 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9560 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9561 break;
9562 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9563 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9564 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9565 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9566 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9567
9568 /* Instruction specific VM-exits: */
9569 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9570 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9571 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9572 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9573 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9574 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9575 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9576 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9577 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9578 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9579 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9580 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9581 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9582 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9583 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9584 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9585 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9586 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9587 case VMX_EXIT_MOV_CRX:
9588 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9589/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9590* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9591 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9592 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9593 SET_BOTH(CRX_READ);
9594 else
9595 SET_BOTH(CRX_WRITE);
9596 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9597 break;
9598 case VMX_EXIT_MOV_DRX:
9599 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9600 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9601 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9602 SET_BOTH(DRX_READ);
9603 else
9604 SET_BOTH(DRX_WRITE);
9605 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9606 break;
9607 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9608 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9609 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9610 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9611 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9612 case VMX_EXIT_XDTR_ACCESS:
9613 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9614 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9615 {
9616 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9617 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9618 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9619 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9620 }
9621 break;
9622
9623 case VMX_EXIT_TR_ACCESS:
9624 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9625 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9626 {
9627 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9628 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9629 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9630 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9631 }
9632 break;
9633
9634 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9635 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9636 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9637 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9638 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9639 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9640 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9641 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9642 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9643 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9644 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9645
9646 /* Events that aren't relevant at this point. */
9647 case VMX_EXIT_EXT_INT:
9648 case VMX_EXIT_INT_WINDOW:
9649 case VMX_EXIT_NMI_WINDOW:
9650 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9651 case VMX_EXIT_PREEMPT_TIMER:
9652 case VMX_EXIT_IO_INSTR:
9653 break;
9654
9655 /* Errors and unexpected events. */
9656 case VMX_EXIT_INIT_SIGNAL:
9657 case VMX_EXIT_SIPI:
9658 case VMX_EXIT_IO_SMI:
9659 case VMX_EXIT_SMI:
9660 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9661 case VMX_EXIT_ERR_MSR_LOAD:
9662 case VMX_EXIT_ERR_MACHINE_CHECK:
9663 break;
9664
9665 default:
9666 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9667 break;
9668 }
9669#undef SET_BOTH
9670#undef SET_EXIT
9671
9672 /*
9673 * Dtrace tracepoints go first. We do them here at once so we don't
9674 * have to copy the guest state saving and stuff a few dozen times.
9675 * Down side is that we've got to repeat the switch, though this time
9676 * we use enmEvent since the probes are a subset of what DBGF does.
9677 */
9678 if (fDtrace1 || fDtrace2)
9679 {
9680 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9681 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9682 switch (enmEvent1)
9683 {
9684 /** @todo consider which extra parameters would be helpful for each probe. */
9685 case DBGFEVENT_END: break;
9686 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9688 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9689 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9690 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9691 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9692 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9693 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9694 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9695 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9696 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9697 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9698 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9699 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9700 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9701 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9703 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9704 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9705 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9706 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9707 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9708 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9709 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9710 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9711 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9712 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9713 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9714 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9715 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9716 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9717 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9718 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9719 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9720 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9721 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9722 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9723 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9724 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9725 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9726 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9727 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9728 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9729 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9730 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9731 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9732 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9733 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9734 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9735 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9736 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9737 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9738 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9739 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9740 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9741 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9742 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9743 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9744 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9745 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9746 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9747 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9748 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9749 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9750 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9751 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9752 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9753 }
9754 switch (enmEvent2)
9755 {
9756 /** @todo consider which extra parameters would be helpful for each probe. */
9757 case DBGFEVENT_END: break;
9758 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9760 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9761 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9762 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9763 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9764 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9765 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9766 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9767 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9768 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9769 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9770 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9771 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9772 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9773 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9774 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9780 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9781 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9782 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9783 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9784 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9810 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9811 }
9812 }
9813
9814 /*
9815 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9816 * the DBGF call will do a full check).
9817 *
9818 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9819 * Note! If we have to events, we prioritize the first, i.e. the instruction
9820 * one, in order to avoid event nesting.
9821 */
9822 if ( enmEvent1 != DBGFEVENT_END
9823 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9824 {
9825 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9826 if (rcStrict != VINF_SUCCESS)
9827 return rcStrict;
9828 }
9829 else if ( enmEvent2 != DBGFEVENT_END
9830 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9831 {
9832 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9833 if (rcStrict != VINF_SUCCESS)
9834 return rcStrict;
9835 }
9836
9837 return VINF_SUCCESS;
9838}
9839
9840
9841/**
9842 * Single-stepping VM-exit filtering.
9843 *
9844 * This is preprocessing the exits and deciding whether we've gotten far enough
9845 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9846 * performed.
9847 *
9848 * @returns Strict VBox status code (i.e. informational status codes too).
9849 * @param pVM The cross context VM structure.
9850 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9851 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9852 * out-of-sync. Make sure to update the required
9853 * fields before using them.
9854 * @param pVmxTransient Pointer to the VMX-transient structure.
9855 * @param uExitReason The VM-exit reason.
9856 * @param pDbgState The debug state.
9857 */
9858DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9859 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9860{
9861 /*
9862 * Expensive (saves context) generic dtrace exit probe.
9863 */
9864 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9865 { /* more likely */ }
9866 else
9867 {
9868 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9869 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9870 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9871 }
9872
9873 /*
9874 * Check for host NMI, just to get that out of the way.
9875 */
9876 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9877 { /* normally likely */ }
9878 else
9879 {
9880 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9881 AssertRCReturn(rc2, rc2);
9882 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9883 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9884 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9885 }
9886
9887 /*
9888 * Check for single stepping event if we're stepping.
9889 */
9890 if (pVCpu->hm.s.fSingleInstruction)
9891 {
9892 switch (uExitReason)
9893 {
9894 case VMX_EXIT_MTF:
9895 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9896
9897 /* Various events: */
9898 case VMX_EXIT_XCPT_OR_NMI:
9899 case VMX_EXIT_EXT_INT:
9900 case VMX_EXIT_TRIPLE_FAULT:
9901 case VMX_EXIT_INT_WINDOW:
9902 case VMX_EXIT_NMI_WINDOW:
9903 case VMX_EXIT_TASK_SWITCH:
9904 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9905 case VMX_EXIT_APIC_ACCESS:
9906 case VMX_EXIT_EPT_VIOLATION:
9907 case VMX_EXIT_EPT_MISCONFIG:
9908 case VMX_EXIT_PREEMPT_TIMER:
9909
9910 /* Instruction specific VM-exits: */
9911 case VMX_EXIT_CPUID:
9912 case VMX_EXIT_GETSEC:
9913 case VMX_EXIT_HLT:
9914 case VMX_EXIT_INVD:
9915 case VMX_EXIT_INVLPG:
9916 case VMX_EXIT_RDPMC:
9917 case VMX_EXIT_RDTSC:
9918 case VMX_EXIT_RSM:
9919 case VMX_EXIT_VMCALL:
9920 case VMX_EXIT_VMCLEAR:
9921 case VMX_EXIT_VMLAUNCH:
9922 case VMX_EXIT_VMPTRLD:
9923 case VMX_EXIT_VMPTRST:
9924 case VMX_EXIT_VMREAD:
9925 case VMX_EXIT_VMRESUME:
9926 case VMX_EXIT_VMWRITE:
9927 case VMX_EXIT_VMXOFF:
9928 case VMX_EXIT_VMXON:
9929 case VMX_EXIT_MOV_CRX:
9930 case VMX_EXIT_MOV_DRX:
9931 case VMX_EXIT_IO_INSTR:
9932 case VMX_EXIT_RDMSR:
9933 case VMX_EXIT_WRMSR:
9934 case VMX_EXIT_MWAIT:
9935 case VMX_EXIT_MONITOR:
9936 case VMX_EXIT_PAUSE:
9937 case VMX_EXIT_XDTR_ACCESS:
9938 case VMX_EXIT_TR_ACCESS:
9939 case VMX_EXIT_INVEPT:
9940 case VMX_EXIT_RDTSCP:
9941 case VMX_EXIT_INVVPID:
9942 case VMX_EXIT_WBINVD:
9943 case VMX_EXIT_XSETBV:
9944 case VMX_EXIT_RDRAND:
9945 case VMX_EXIT_INVPCID:
9946 case VMX_EXIT_VMFUNC:
9947 case VMX_EXIT_RDSEED:
9948 case VMX_EXIT_XSAVES:
9949 case VMX_EXIT_XRSTORS:
9950 {
9951 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9952 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9953 AssertRCReturn(rc2, rc2);
9954 if ( pMixedCtx->rip != pDbgState->uRipStart
9955 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9956 return VINF_EM_DBG_STEPPED;
9957 break;
9958 }
9959
9960 /* Errors and unexpected events: */
9961 case VMX_EXIT_INIT_SIGNAL:
9962 case VMX_EXIT_SIPI:
9963 case VMX_EXIT_IO_SMI:
9964 case VMX_EXIT_SMI:
9965 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9966 case VMX_EXIT_ERR_MSR_LOAD:
9967 case VMX_EXIT_ERR_MACHINE_CHECK:
9968 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9969 break;
9970
9971 default:
9972 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9973 break;
9974 }
9975 }
9976
9977 /*
9978 * Check for debugger event breakpoints and dtrace probes.
9979 */
9980 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9981 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9982 {
9983 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9984 if (rcStrict != VINF_SUCCESS)
9985 return rcStrict;
9986 }
9987
9988 /*
9989 * Normal processing.
9990 */
9991#ifdef HMVMX_USE_FUNCTION_TABLE
9992 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9993#else
9994 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9995#endif
9996}
9997
9998
9999/**
10000 * Single steps guest code using VT-x.
10001 *
10002 * @returns Strict VBox status code (i.e. informational status codes too).
10003 * @param pVM The cross context VM structure.
10004 * @param pVCpu The cross context virtual CPU structure.
10005 * @param pCtx Pointer to the guest-CPU context.
10006 *
10007 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10008 */
10009static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10010{
10011 VMXTRANSIENT VmxTransient;
10012 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10013
10014 /* Set HMCPU indicators. */
10015 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10016 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10017 pVCpu->hm.s.fDebugWantRdTscExit = false;
10018 pVCpu->hm.s.fUsingDebugLoop = true;
10019
10020 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10021 VMXRUNDBGSTATE DbgState;
10022 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10023 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10024
10025 /*
10026 * The loop.
10027 */
10028 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10029 for (uint32_t cLoops = 0; ; cLoops++)
10030 {
10031 Assert(!HMR0SuspendPending());
10032 HMVMX_ASSERT_CPU_SAFE();
10033 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10034
10035 /*
10036 * Preparatory work for running guest code, this may force us to return
10037 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10038 */
10039 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10040 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10041 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10042 if (rcStrict != VINF_SUCCESS)
10043 break;
10044
10045 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10046 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10047
10048 /*
10049 * Now we can run the guest code.
10050 */
10051 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10052
10053 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10054
10055 /*
10056 * Restore any residual host-state and save any bits shared between host
10057 * and guest into the guest-CPU state. Re-enables interrupts!
10058 */
10059 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10060
10061 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10062 if (RT_SUCCESS(rcRun))
10063 { /* very likely */ }
10064 else
10065 {
10066 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10067 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10068 return rcRun;
10069 }
10070
10071 /* Profile the VM-exit. */
10072 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10073 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10074 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10075 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10076 HMVMX_START_EXIT_DISPATCH_PROF();
10077
10078 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10079
10080 /*
10081 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10082 */
10083 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10084 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10085 if (rcStrict != VINF_SUCCESS)
10086 break;
10087 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10088 {
10089 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10090 rcStrict = VINF_EM_RAW_INTERRUPT;
10091 break;
10092 }
10093
10094 /*
10095 * Stepping: Did the RIP change, if so, consider it a single step.
10096 * Otherwise, make sure one of the TFs gets set.
10097 */
10098 if (fStepping)
10099 {
10100 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10101 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10102 AssertRCReturn(rc2, rc2);
10103 if ( pCtx->rip != DbgState.uRipStart
10104 || pCtx->cs.Sel != DbgState.uCsStart)
10105 {
10106 rcStrict = VINF_EM_DBG_STEPPED;
10107 break;
10108 }
10109 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10110 }
10111
10112 /*
10113 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10114 */
10115 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10116 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10117 }
10118
10119 /*
10120 * Clear the X86_EFL_TF if necessary.
10121 */
10122 if (pVCpu->hm.s.fClearTrapFlag)
10123 {
10124 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10125 AssertRCReturn(rc2, rc2);
10126 pVCpu->hm.s.fClearTrapFlag = false;
10127 pCtx->eflags.Bits.u1TF = 0;
10128 }
10129 /** @todo there seems to be issues with the resume flag when the monitor trap
10130 * flag is pending without being used. Seen early in bios init when
10131 * accessing APIC page in protected mode. */
10132
10133 /*
10134 * Restore VM-exit control settings as we may not reenter this function the
10135 * next time around.
10136 */
10137 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10138
10139 /* Restore HMCPU indicators. */
10140 pVCpu->hm.s.fUsingDebugLoop = false;
10141 pVCpu->hm.s.fDebugWantRdTscExit = false;
10142 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10143
10144 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10145 return rcStrict;
10146}
10147
10148
10149/** @} */
10150
10151
10152/**
10153 * Checks if any expensive dtrace probes are enabled and we should go to the
10154 * debug loop.
10155 *
10156 * @returns true if we should use debug loop, false if not.
10157 */
10158static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10159{
10160 /* It's probably faster to OR the raw 32-bit counter variables together.
10161 Since the variables are in an array and the probes are next to one
10162 another (more or less), we have good locality. So, better read
10163 eight-nine cache lines ever time and only have one conditional, than
10164 128+ conditionals, right? */
10165 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10166 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10167 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10168 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10169 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10170 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10171 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10172 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10173 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10174 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10175 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10176 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10177 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10178 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10179 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10180 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10181 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10182 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10183 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10184 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10185 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10186 ) != 0
10187 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10188 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10189 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10190 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10191 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10192 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10193 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10194 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10195 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10196 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10197 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10198 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10199 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10200 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10201 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10202 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10203 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10204 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10205 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10206 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10207 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10208 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10209 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10210 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10211 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10212 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10213 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10214 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10215 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10216 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10217 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10218 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10219 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10220 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10221 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10222 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10223 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10224 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10225 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10226 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10227 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10228 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10229 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10230 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10231 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10232 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10233 ) != 0
10234 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10235 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10236 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10237 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10238 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10239 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10240 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10241 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10242 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10243 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10244 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10245 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10246 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10247 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10248 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10249 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10250 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10251 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10252 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10253 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10254 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10255 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10256 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10257 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10258 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10259 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10260 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10261 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10262 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10263 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10264 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10265 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10266 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10267 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10268 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10269 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10270 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10271 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10272 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10273 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10274 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10275 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10279 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10280 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10281 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10282 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10283 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10284 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10285 ) != 0;
10286}
10287
10288
10289/**
10290 * Runs the guest code using VT-x.
10291 *
10292 * @returns Strict VBox status code (i.e. informational status codes too).
10293 * @param pVM The cross context VM structure.
10294 * @param pVCpu The cross context virtual CPU structure.
10295 * @param pCtx Pointer to the guest-CPU context.
10296 */
10297VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10298{
10299 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10300 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10301 HMVMX_ASSERT_PREEMPT_SAFE();
10302
10303 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10304
10305 VBOXSTRICTRC rcStrict;
10306 if ( !pVCpu->hm.s.fUseDebugLoop
10307 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10308 && !DBGFIsStepping(pVCpu) )
10309 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10310 else
10311 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10312
10313 if (rcStrict == VERR_EM_INTERPRETER)
10314 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10315 else if (rcStrict == VINF_EM_RESET)
10316 rcStrict = VINF_EM_TRIPLE_FAULT;
10317
10318 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10319 if (RT_FAILURE(rc2))
10320 {
10321 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10322 rcStrict = rc2;
10323 }
10324 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10325 return rcStrict;
10326}
10327
10328
10329#ifndef HMVMX_USE_FUNCTION_TABLE
10330DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10331{
10332# ifdef DEBUG_ramshankar
10333# define RETURN_EXIT_CALL(a_CallExpr) \
10334 do { \
10335 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10336 VBOXSTRICTRC rcStrict = a_CallExpr; \
10337 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10338 return rcStrict; \
10339 } while (0)
10340# else
10341# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10342# endif
10343 switch (rcReason)
10344 {
10345 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10346 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10347 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10348 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10349 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10350 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10351 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10352 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10353 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10354 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10355 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10356 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10357 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10358 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10359 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10360 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10361 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10362 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10363 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10364 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10365 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10366 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10367 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10368 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10369 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10370 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10371 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10372 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10373 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10374 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10375 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10376 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10377 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10378 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10379
10380 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10381 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10382 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10383 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10384 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10385 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10386 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10387 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10388 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10389
10390 case VMX_EXIT_VMCLEAR:
10391 case VMX_EXIT_VMLAUNCH:
10392 case VMX_EXIT_VMPTRLD:
10393 case VMX_EXIT_VMPTRST:
10394 case VMX_EXIT_VMREAD:
10395 case VMX_EXIT_VMRESUME:
10396 case VMX_EXIT_VMWRITE:
10397 case VMX_EXIT_VMXOFF:
10398 case VMX_EXIT_VMXON:
10399 case VMX_EXIT_INVEPT:
10400 case VMX_EXIT_INVVPID:
10401 case VMX_EXIT_VMFUNC:
10402 case VMX_EXIT_XSAVES:
10403 case VMX_EXIT_XRSTORS:
10404 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10405 case VMX_EXIT_RESERVED_60:
10406 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10407 case VMX_EXIT_RESERVED_62:
10408 default:
10409 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10410 }
10411#undef RETURN_EXIT_CALL
10412}
10413#endif /* !HMVMX_USE_FUNCTION_TABLE */
10414
10415
10416#ifdef VBOX_STRICT
10417/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10418# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10419 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10420
10421# define HMVMX_ASSERT_PREEMPT_CPUID() \
10422 do { \
10423 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10424 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10425 } while (0)
10426
10427# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10428 do { \
10429 AssertPtr(pVCpu); \
10430 AssertPtr(pMixedCtx); \
10431 AssertPtr(pVmxTransient); \
10432 Assert(pVmxTransient->fVMEntryFailed == false); \
10433 Assert(ASMIntAreEnabled()); \
10434 HMVMX_ASSERT_PREEMPT_SAFE(); \
10435 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10436 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10437 HMVMX_ASSERT_PREEMPT_SAFE(); \
10438 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10439 HMVMX_ASSERT_PREEMPT_CPUID(); \
10440 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10441 } while (0)
10442
10443# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10444 do { \
10445 Log4Func(("\n")); \
10446 } while (0)
10447#else /* nonstrict builds: */
10448# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10449 do { \
10450 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10451 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10452 } while (0)
10453# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10454#endif
10455
10456
10457/**
10458 * Advances the guest RIP by the specified number of bytes.
10459 *
10460 * @param pVCpu The cross context virtual CPU structure.
10461 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10462 * out-of-sync. Make sure to update the required fields
10463 * before using them.
10464 * @param cbInstr Number of bytes to advance the RIP by.
10465 *
10466 * @remarks No-long-jump zone!!!
10467 */
10468DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10469{
10470 /* Advance the RIP. */
10471 pMixedCtx->rip += cbInstr;
10472 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10473
10474 /* Update interrupt inhibition. */
10475 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10476 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10477 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10478}
10479
10480
10481/**
10482 * Advances the guest RIP after reading it from the VMCS.
10483 *
10484 * @returns VBox status code, no informational status codes.
10485 * @param pVCpu The cross context virtual CPU structure.
10486 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10487 * out-of-sync. Make sure to update the required fields
10488 * before using them.
10489 * @param pVmxTransient Pointer to the VMX transient structure.
10490 *
10491 * @remarks No-long-jump zone!!!
10492 */
10493static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10494{
10495 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10496 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10497 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10498 AssertRCReturn(rc, rc);
10499
10500 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10501
10502 /*
10503 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10504 * pending debug exception field as it takes care of priority of events.
10505 *
10506 * See Intel spec. 32.2.1 "Debug Exceptions".
10507 */
10508 if ( !pVCpu->hm.s.fSingleInstruction
10509 && pMixedCtx->eflags.Bits.u1TF)
10510 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10511
10512 return VINF_SUCCESS;
10513}
10514
10515
10516/**
10517 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10518 * and update error record fields accordingly.
10519 *
10520 * @return VMX_IGS_* return codes.
10521 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10522 * wrong with the guest state.
10523 *
10524 * @param pVM The cross context VM structure.
10525 * @param pVCpu The cross context virtual CPU structure.
10526 * @param pCtx Pointer to the guest-CPU state.
10527 *
10528 * @remarks This function assumes our cache of the VMCS controls
10529 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10530 */
10531static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10532{
10533#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10534#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10535 uError = (err); \
10536 break; \
10537 } else do { } while (0)
10538
10539 int rc;
10540 uint32_t uError = VMX_IGS_ERROR;
10541 uint32_t u32Val;
10542 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10543
10544 do
10545 {
10546 /*
10547 * CR0.
10548 */
10549 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10550 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10551 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10552 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10553 if (fUnrestrictedGuest)
10554 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10555
10556 uint32_t u32GuestCR0;
10557 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10558 AssertRCBreak(rc);
10559 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10560 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10561 if ( !fUnrestrictedGuest
10562 && (u32GuestCR0 & X86_CR0_PG)
10563 && !(u32GuestCR0 & X86_CR0_PE))
10564 {
10565 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10566 }
10567
10568 /*
10569 * CR4.
10570 */
10571 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10572 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10573
10574 uint32_t u32GuestCR4;
10575 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10576 AssertRCBreak(rc);
10577 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10578 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10579
10580 /*
10581 * IA32_DEBUGCTL MSR.
10582 */
10583 uint64_t u64Val;
10584 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10585 AssertRCBreak(rc);
10586 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10587 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10588 {
10589 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10590 }
10591 uint64_t u64DebugCtlMsr = u64Val;
10592
10593#ifdef VBOX_STRICT
10594 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10595 AssertRCBreak(rc);
10596 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10597#endif
10598 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10599
10600 /*
10601 * RIP and RFLAGS.
10602 */
10603 uint32_t u32Eflags;
10604#if HC_ARCH_BITS == 64
10605 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10606 AssertRCBreak(rc);
10607 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10608 if ( !fLongModeGuest
10609 || !pCtx->cs.Attr.n.u1Long)
10610 {
10611 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10612 }
10613 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10614 * must be identical if the "IA-32e mode guest" VM-entry
10615 * control is 1 and CS.L is 1. No check applies if the
10616 * CPU supports 64 linear-address bits. */
10617
10618 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10619 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10620 AssertRCBreak(rc);
10621 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10622 VMX_IGS_RFLAGS_RESERVED);
10623 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10624 u32Eflags = u64Val;
10625#else
10626 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10627 AssertRCBreak(rc);
10628 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10629 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10630#endif
10631
10632 if ( fLongModeGuest
10633 || ( fUnrestrictedGuest
10634 && !(u32GuestCR0 & X86_CR0_PE)))
10635 {
10636 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10637 }
10638
10639 uint32_t u32EntryInfo;
10640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10641 AssertRCBreak(rc);
10642 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10643 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10644 {
10645 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10646 }
10647
10648 /*
10649 * 64-bit checks.
10650 */
10651#if HC_ARCH_BITS == 64
10652 if (fLongModeGuest)
10653 {
10654 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10655 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10656 }
10657
10658 if ( !fLongModeGuest
10659 && (u32GuestCR4 & X86_CR4_PCIDE))
10660 {
10661 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10662 }
10663
10664 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10665 * 51:32 beyond the processor's physical-address width are 0. */
10666
10667 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10668 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10669 {
10670 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10671 }
10672
10673 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10674 AssertRCBreak(rc);
10675 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10676
10677 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10678 AssertRCBreak(rc);
10679 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10680#endif
10681
10682 /*
10683 * PERF_GLOBAL MSR.
10684 */
10685 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10686 {
10687 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10688 AssertRCBreak(rc);
10689 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10690 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10691 }
10692
10693 /*
10694 * PAT MSR.
10695 */
10696 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10697 {
10698 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10699 AssertRCBreak(rc);
10700 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10701 for (unsigned i = 0; i < 8; i++)
10702 {
10703 uint8_t u8Val = (u64Val & 0xff);
10704 if ( u8Val != 0 /* UC */
10705 && u8Val != 1 /* WC */
10706 && u8Val != 4 /* WT */
10707 && u8Val != 5 /* WP */
10708 && u8Val != 6 /* WB */
10709 && u8Val != 7 /* UC- */)
10710 {
10711 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10712 }
10713 u64Val >>= 8;
10714 }
10715 }
10716
10717 /*
10718 * EFER MSR.
10719 */
10720 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10721 {
10722 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10723 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10724 AssertRCBreak(rc);
10725 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10726 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10727 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10728 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10729 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10730 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10731 || !(u32GuestCR0 & X86_CR0_PG)
10732 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10733 VMX_IGS_EFER_LMA_LME_MISMATCH);
10734 }
10735
10736 /*
10737 * Segment registers.
10738 */
10739 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10740 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10741 if (!(u32Eflags & X86_EFL_VM))
10742 {
10743 /* CS */
10744 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10745 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10746 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10747 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10748 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10749 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10750 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10751 /* CS cannot be loaded with NULL in protected mode. */
10752 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10753 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10754 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10755 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10756 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10757 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10758 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10759 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10760 else
10761 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10762
10763 /* SS */
10764 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10765 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10766 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10767 if ( !(pCtx->cr0 & X86_CR0_PE)
10768 || pCtx->cs.Attr.n.u4Type == 3)
10769 {
10770 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10771 }
10772 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10773 {
10774 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10775 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10776 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10777 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10778 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10779 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10780 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10781 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10782 }
10783
10784 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10785 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10786 {
10787 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10788 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10789 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10790 || pCtx->ds.Attr.n.u4Type > 11
10791 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10792 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10793 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10794 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10795 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10796 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10797 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10798 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10799 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10800 }
10801 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10802 {
10803 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10804 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10805 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10806 || pCtx->es.Attr.n.u4Type > 11
10807 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10808 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10809 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10810 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10811 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10812 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10813 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10814 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10815 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10816 }
10817 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10818 {
10819 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10820 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10821 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10822 || pCtx->fs.Attr.n.u4Type > 11
10823 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10824 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10825 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10826 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10827 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10828 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10829 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10830 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10831 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10832 }
10833 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10834 {
10835 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10836 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10837 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10838 || pCtx->gs.Attr.n.u4Type > 11
10839 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10840 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10841 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10842 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10843 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10844 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10845 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10846 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10847 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10848 }
10849 /* 64-bit capable CPUs. */
10850#if HC_ARCH_BITS == 64
10851 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10852 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10853 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10854 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10855 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10856 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10857 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10858 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10859 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10860 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10861 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10862#endif
10863 }
10864 else
10865 {
10866 /* V86 mode checks. */
10867 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10868 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10869 {
10870 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10871 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10872 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10873 }
10874 else
10875 {
10876 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10877 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10878 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10879 }
10880
10881 /* CS */
10882 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10883 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10884 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10885 /* SS */
10886 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10887 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10888 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10889 /* DS */
10890 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10891 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10892 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10893 /* ES */
10894 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10895 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10896 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10897 /* FS */
10898 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10899 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10900 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10901 /* GS */
10902 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10903 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10904 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10905 /* 64-bit capable CPUs. */
10906#if HC_ARCH_BITS == 64
10907 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10908 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10909 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10910 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10911 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10912 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10913 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10914 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10915 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10916 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10917 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10918#endif
10919 }
10920
10921 /*
10922 * TR.
10923 */
10924 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10925 /* 64-bit capable CPUs. */
10926#if HC_ARCH_BITS == 64
10927 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10928#endif
10929 if (fLongModeGuest)
10930 {
10931 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10932 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10933 }
10934 else
10935 {
10936 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10937 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10938 VMX_IGS_TR_ATTR_TYPE_INVALID);
10939 }
10940 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10941 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10942 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10943 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10944 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10945 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10946 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10947 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10948
10949 /*
10950 * GDTR and IDTR.
10951 */
10952#if HC_ARCH_BITS == 64
10953 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10954 AssertRCBreak(rc);
10955 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10956
10957 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10958 AssertRCBreak(rc);
10959 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10960#endif
10961
10962 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10963 AssertRCBreak(rc);
10964 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10965
10966 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10967 AssertRCBreak(rc);
10968 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10969
10970 /*
10971 * Guest Non-Register State.
10972 */
10973 /* Activity State. */
10974 uint32_t u32ActivityState;
10975 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10976 AssertRCBreak(rc);
10977 HMVMX_CHECK_BREAK( !u32ActivityState
10978 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10979 VMX_IGS_ACTIVITY_STATE_INVALID);
10980 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10981 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10982 uint32_t u32IntrState;
10983 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10984 AssertRCBreak(rc);
10985 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10986 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10987 {
10988 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10989 }
10990
10991 /** @todo Activity state and injecting interrupts. Left as a todo since we
10992 * currently don't use activity states but ACTIVE. */
10993
10994 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10995 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10996
10997 /* Guest interruptibility-state. */
10998 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10999 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11000 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11001 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11002 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11003 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11004 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11005 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11006 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11007 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11008 {
11009 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11010 {
11011 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11012 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11013 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11014 }
11015 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11016 {
11017 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11018 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11019 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11020 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11021 }
11022 }
11023 /** @todo Assumes the processor is not in SMM. */
11024 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11025 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11026 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11027 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11028 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11029 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11030 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11031 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11032 {
11033 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11034 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11035 }
11036
11037 /* Pending debug exceptions. */
11038#if HC_ARCH_BITS == 64
11039 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11040 AssertRCBreak(rc);
11041 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11042 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11043 u32Val = u64Val; /* For pending debug exceptions checks below. */
11044#else
11045 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11046 AssertRCBreak(rc);
11047 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11048 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11049#endif
11050
11051 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11052 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11053 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11054 {
11055 if ( (u32Eflags & X86_EFL_TF)
11056 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11057 {
11058 /* Bit 14 is PendingDebug.BS. */
11059 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11060 }
11061 if ( !(u32Eflags & X86_EFL_TF)
11062 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11063 {
11064 /* Bit 14 is PendingDebug.BS. */
11065 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11066 }
11067 }
11068
11069 /* VMCS link pointer. */
11070 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11071 AssertRCBreak(rc);
11072 if (u64Val != UINT64_C(0xffffffffffffffff))
11073 {
11074 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11075 /** @todo Bits beyond the processor's physical-address width MBZ. */
11076 /** @todo 32-bit located in memory referenced by value of this field (as a
11077 * physical address) must contain the processor's VMCS revision ID. */
11078 /** @todo SMM checks. */
11079 }
11080
11081 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11082 * not using Nested Paging? */
11083 if ( pVM->hm.s.fNestedPaging
11084 && !fLongModeGuest
11085 && CPUMIsGuestInPAEModeEx(pCtx))
11086 {
11087 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11088 AssertRCBreak(rc);
11089 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11090
11091 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11092 AssertRCBreak(rc);
11093 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11094
11095 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11096 AssertRCBreak(rc);
11097 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11098
11099 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11100 AssertRCBreak(rc);
11101 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11102 }
11103
11104 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11105 if (uError == VMX_IGS_ERROR)
11106 uError = VMX_IGS_REASON_NOT_FOUND;
11107 } while (0);
11108
11109 pVCpu->hm.s.u32HMError = uError;
11110 return uError;
11111
11112#undef HMVMX_ERROR_BREAK
11113#undef HMVMX_CHECK_BREAK
11114}
11115
11116/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11117/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11118/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11119
11120/** @name VM-exit handlers.
11121 * @{
11122 */
11123
11124/**
11125 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11126 */
11127HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11128{
11129 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11130 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11131 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11132 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11133 return VINF_SUCCESS;
11134 return VINF_EM_RAW_INTERRUPT;
11135}
11136
11137
11138/**
11139 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11140 */
11141HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11142{
11143 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11144 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11145
11146 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11147 AssertRCReturn(rc, rc);
11148
11149 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11150 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11151 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11152 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11153
11154 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11155 {
11156 /*
11157 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11158 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11159 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11160 *
11161 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11162 */
11163 VMXDispatchHostNmi();
11164 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11165 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11166 return VINF_SUCCESS;
11167 }
11168
11169 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11170 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11171 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11172 { /* likely */ }
11173 else
11174 {
11175 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11176 rcStrictRc1 = VINF_SUCCESS;
11177 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11178 return rcStrictRc1;
11179 }
11180
11181 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11182 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11183 switch (uIntType)
11184 {
11185 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11186 Assert(uVector == X86_XCPT_DB);
11187 /* no break */
11188 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11189 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11190 /* no break */
11191 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11192 {
11193 /*
11194 * If there's any exception caused as a result of event injection, go back to
11195 * the interpreter. The page-fault case is complicated and we manually handle
11196 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11197 * handled in hmR0VmxCheckExitDueToEventDelivery.
11198 */
11199 if (!pVCpu->hm.s.Event.fPending)
11200 { /* likely */ }
11201 else if ( uVector != X86_XCPT_PF
11202 && uVector != X86_XCPT_AC)
11203 {
11204 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11205 rc = VERR_EM_INTERPRETER;
11206 break;
11207 }
11208
11209 switch (uVector)
11210 {
11211 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11212 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11213 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11214 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11215 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11216 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11217 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11218
11219 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11220 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11221 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11222 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11223 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11224 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11225 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11226 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11227 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11228 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11229 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11230 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11231 default:
11232 {
11233 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11234 AssertRCReturn(rc, rc);
11235
11236 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11237 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11238 {
11239 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11240 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11241 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11242
11243 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11244 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11245 AssertRCReturn(rc, rc);
11246 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11247 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11248 0 /* GCPtrFaultAddress */);
11249 AssertRCReturn(rc, rc);
11250 }
11251 else
11252 {
11253 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11254 pVCpu->hm.s.u32HMError = uVector;
11255 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11256 }
11257 break;
11258 }
11259 }
11260 break;
11261 }
11262
11263 default:
11264 {
11265 pVCpu->hm.s.u32HMError = uExitIntInfo;
11266 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11267 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11268 break;
11269 }
11270 }
11271 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11272 return rc;
11273}
11274
11275
11276/**
11277 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11278 */
11279HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11280{
11281 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11282
11283 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11284 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11285
11286 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11288 return VINF_SUCCESS;
11289}
11290
11291
11292/**
11293 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11294 */
11295HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11296{
11297 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11298 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11299 {
11300 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11301 HMVMX_RETURN_UNEXPECTED_EXIT();
11302 }
11303
11304 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11305
11306 /*
11307 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11308 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11309 */
11310 uint32_t uIntrState = 0;
11311 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11312 AssertRCReturn(rc, rc);
11313
11314 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11315 if ( fBlockSti
11316 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11317 {
11318 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11319 }
11320
11321 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11322 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11323
11324 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11325 return VINF_SUCCESS;
11326}
11327
11328
11329/**
11330 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11331 */
11332HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11333{
11334 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11336 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11337}
11338
11339
11340/**
11341 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11342 */
11343HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11344{
11345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11347 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11348}
11349
11350
11351/**
11352 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11353 */
11354HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11355{
11356 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11357 PVM pVM = pVCpu->CTX_SUFF(pVM);
11358 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11359 if (RT_LIKELY(rc == VINF_SUCCESS))
11360 {
11361 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11362 Assert(pVmxTransient->cbInstr == 2);
11363 }
11364 else
11365 {
11366 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11367 rc = VERR_EM_INTERPRETER;
11368 }
11369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11370 return rc;
11371}
11372
11373
11374/**
11375 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11376 */
11377HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11378{
11379 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11380 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11381 AssertRCReturn(rc, rc);
11382
11383 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11384 return VINF_EM_RAW_EMULATE_INSTR;
11385
11386 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11387 HMVMX_RETURN_UNEXPECTED_EXIT();
11388}
11389
11390
11391/**
11392 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11393 */
11394HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11395{
11396 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11397 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11398 AssertRCReturn(rc, rc);
11399
11400 PVM pVM = pVCpu->CTX_SUFF(pVM);
11401 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11402 if (RT_LIKELY(rc == VINF_SUCCESS))
11403 {
11404 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11405 Assert(pVmxTransient->cbInstr == 2);
11406 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11407 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11408 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11409 }
11410 else
11411 rc = VERR_EM_INTERPRETER;
11412 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11413 return rc;
11414}
11415
11416
11417/**
11418 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11419 */
11420HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11421{
11422 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11423 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11424 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11425 AssertRCReturn(rc, rc);
11426
11427 PVM pVM = pVCpu->CTX_SUFF(pVM);
11428 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11429 if (RT_SUCCESS(rc))
11430 {
11431 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11432 Assert(pVmxTransient->cbInstr == 3);
11433 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11434 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11435 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11436 }
11437 else
11438 {
11439 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11440 rc = VERR_EM_INTERPRETER;
11441 }
11442 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11443 return rc;
11444}
11445
11446
11447/**
11448 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11449 */
11450HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11451{
11452 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11453 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11454 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11455 AssertRCReturn(rc, rc);
11456
11457 PVM pVM = pVCpu->CTX_SUFF(pVM);
11458 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11459 if (RT_LIKELY(rc == VINF_SUCCESS))
11460 {
11461 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11462 Assert(pVmxTransient->cbInstr == 2);
11463 }
11464 else
11465 {
11466 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11467 rc = VERR_EM_INTERPRETER;
11468 }
11469 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11470 return rc;
11471}
11472
11473
11474/**
11475 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11476 */
11477HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11478{
11479 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11481
11482 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11483 if (pVCpu->hm.s.fHypercallsEnabled)
11484 {
11485#if 0
11486 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11487#else
11488 /* Aggressive state sync. for now. */
11489 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11490 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11491 AssertRCReturn(rc, rc);
11492#endif
11493
11494 /* Perform the hypercall. */
11495 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11496 if (rcStrict == VINF_SUCCESS)
11497 {
11498 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11499 AssertRCReturn(rc, rc);
11500 }
11501 else
11502 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11503 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11504 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11505
11506 /* If the hypercall changes anything other than guest's general-purpose registers,
11507 we would need to reload the guest changed bits here before VM-entry. */
11508 }
11509 else
11510 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11511
11512 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11513 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11514 {
11515 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11516 rcStrict = VINF_SUCCESS;
11517 }
11518
11519 return rcStrict;
11520}
11521
11522
11523/**
11524 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11525 */
11526HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11527{
11528 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11529 PVM pVM = pVCpu->CTX_SUFF(pVM);
11530 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11531
11532 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11533 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11534 AssertRCReturn(rc, rc);
11535
11536 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11537 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11538 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11539 else
11540 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11541 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11543 return rcStrict;
11544}
11545
11546
11547/**
11548 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11549 */
11550HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11551{
11552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11553 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11554 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11555 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11556 AssertRCReturn(rc, rc);
11557
11558 PVM pVM = pVCpu->CTX_SUFF(pVM);
11559 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11560 if (RT_LIKELY(rc == VINF_SUCCESS))
11561 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11562 else
11563 {
11564 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11565 rc = VERR_EM_INTERPRETER;
11566 }
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11568 return rc;
11569}
11570
11571
11572/**
11573 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11574 */
11575HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11576{
11577 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11578 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11579 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11580 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11581 AssertRCReturn(rc, rc);
11582
11583 PVM pVM = pVCpu->CTX_SUFF(pVM);
11584 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11585 rc = VBOXSTRICTRC_VAL(rc2);
11586 if (RT_LIKELY( rc == VINF_SUCCESS
11587 || rc == VINF_EM_HALT))
11588 {
11589 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11590 AssertRCReturn(rc3, rc3);
11591
11592 if ( rc == VINF_EM_HALT
11593 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11594 {
11595 rc = VINF_SUCCESS;
11596 }
11597 }
11598 else
11599 {
11600 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11601 rc = VERR_EM_INTERPRETER;
11602 }
11603 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11604 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11605 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11606 return rc;
11607}
11608
11609
11610/**
11611 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11612 */
11613HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11614{
11615 /*
11616 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11617 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11618 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11619 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11620 */
11621 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11622 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11623 HMVMX_RETURN_UNEXPECTED_EXIT();
11624}
11625
11626
11627/**
11628 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11629 */
11630HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11631{
11632 /*
11633 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11634 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11635 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11636 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11637 */
11638 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11639 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11640 HMVMX_RETURN_UNEXPECTED_EXIT();
11641}
11642
11643
11644/**
11645 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11646 */
11647HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11648{
11649 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11650 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11651 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11652 HMVMX_RETURN_UNEXPECTED_EXIT();
11653}
11654
11655
11656/**
11657 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11658 */
11659HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11660{
11661 /*
11662 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11663 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11664 * See Intel spec. 25.3 "Other Causes of VM-exits".
11665 */
11666 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11667 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11668 HMVMX_RETURN_UNEXPECTED_EXIT();
11669}
11670
11671
11672/**
11673 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11674 * VM-exit.
11675 */
11676HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11677{
11678 /*
11679 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11680 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11681 *
11682 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11683 * See Intel spec. "23.8 Restrictions on VMX operation".
11684 */
11685 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11686 return VINF_SUCCESS;
11687}
11688
11689
11690/**
11691 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11692 * VM-exit.
11693 */
11694HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11695{
11696 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11697 return VINF_EM_RESET;
11698}
11699
11700
11701/**
11702 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11703 */
11704HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11705{
11706 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11707 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11708
11709 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11710 AssertRCReturn(rc, rc);
11711
11712 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11713 rc = VINF_SUCCESS;
11714 else
11715 rc = VINF_EM_HALT;
11716
11717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11718 if (rc != VINF_SUCCESS)
11719 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11720 return rc;
11721}
11722
11723
11724/**
11725 * VM-exit handler for instructions that result in a \#UD exception delivered to
11726 * the guest.
11727 */
11728HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11729{
11730 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11731 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11732 return VINF_SUCCESS;
11733}
11734
11735
11736/**
11737 * VM-exit handler for expiry of the VMX preemption timer.
11738 */
11739HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11740{
11741 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11742
11743 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11744 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11745
11746 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11747 PVM pVM = pVCpu->CTX_SUFF(pVM);
11748 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11749 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11750 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11751}
11752
11753
11754/**
11755 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11756 */
11757HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11758{
11759 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11760
11761 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11762 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11763 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11764 AssertRCReturn(rc, rc);
11765
11766 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11767 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11768
11769 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11770
11771 return rcStrict;
11772}
11773
11774
11775/**
11776 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11777 */
11778HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11779{
11780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781
11782 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11783 /** @todo implement EMInterpretInvpcid() */
11784 return VERR_EM_INTERPRETER;
11785}
11786
11787
11788/**
11789 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11790 * Error VM-exit.
11791 */
11792HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11793{
11794 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11795 AssertRCReturn(rc, rc);
11796
11797 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11798 AssertRCReturn(rc, rc);
11799
11800 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11801 NOREF(uInvalidReason);
11802
11803#ifdef VBOX_STRICT
11804 uint32_t uIntrState;
11805 RTHCUINTREG uHCReg;
11806 uint64_t u64Val;
11807 uint32_t u32Val;
11808
11809 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11810 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11811 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11812 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11813 AssertRCReturn(rc, rc);
11814
11815 Log4(("uInvalidReason %u\n", uInvalidReason));
11816 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11817 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11818 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11819 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11820
11821 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11822 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11823 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11824 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11825 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11826 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11827 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11828 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11829 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11830 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11831 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11832 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11833#else
11834 NOREF(pVmxTransient);
11835#endif
11836
11837 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11838 return VERR_VMX_INVALID_GUEST_STATE;
11839}
11840
11841
11842/**
11843 * VM-exit handler for VM-entry failure due to an MSR-load
11844 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11845 */
11846HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11847{
11848 NOREF(pVmxTransient);
11849 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11850 HMVMX_RETURN_UNEXPECTED_EXIT();
11851}
11852
11853
11854/**
11855 * VM-exit handler for VM-entry failure due to a machine-check event
11856 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11857 */
11858HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11859{
11860 NOREF(pVmxTransient);
11861 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11862 HMVMX_RETURN_UNEXPECTED_EXIT();
11863}
11864
11865
11866/**
11867 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11868 * theory.
11869 */
11870HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11871{
11872 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11873 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11874 return VERR_VMX_UNDEFINED_EXIT_CODE;
11875}
11876
11877
11878/**
11879 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11880 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11881 * Conditional VM-exit.
11882 */
11883HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11884{
11885 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11886
11887 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11888 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11889 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11890 return VERR_EM_INTERPRETER;
11891 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11892 HMVMX_RETURN_UNEXPECTED_EXIT();
11893}
11894
11895
11896/**
11897 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11898 */
11899HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11900{
11901 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11902
11903 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11904 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11905 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11906 return VERR_EM_INTERPRETER;
11907 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11908 HMVMX_RETURN_UNEXPECTED_EXIT();
11909}
11910
11911
11912/**
11913 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11914 */
11915HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11916{
11917 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11918
11919 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11920 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11921 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11922 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11923 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11924 {
11925 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11926 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11927 }
11928 AssertRCReturn(rc, rc);
11929 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11930
11931#ifdef VBOX_STRICT
11932 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11933 {
11934 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11935 && pMixedCtx->ecx != MSR_K6_EFER)
11936 {
11937 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11938 pMixedCtx->ecx));
11939 HMVMX_RETURN_UNEXPECTED_EXIT();
11940 }
11941 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11942 {
11943 VMXMSREXITREAD enmRead;
11944 VMXMSREXITWRITE enmWrite;
11945 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11946 AssertRCReturn(rc2, rc2);
11947 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11948 {
11949 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11950 HMVMX_RETURN_UNEXPECTED_EXIT();
11951 }
11952 }
11953 }
11954#endif
11955
11956 PVM pVM = pVCpu->CTX_SUFF(pVM);
11957 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11958 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11959 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11960 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11961 if (RT_SUCCESS(rc))
11962 {
11963 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11964 Assert(pVmxTransient->cbInstr == 2);
11965 }
11966 return rc;
11967}
11968
11969
11970/**
11971 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11972 */
11973HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11974{
11975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11976 PVM pVM = pVCpu->CTX_SUFF(pVM);
11977 int rc = VINF_SUCCESS;
11978
11979 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11980 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11981 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11982 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11983 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11984 {
11985 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11986 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11987 }
11988 AssertRCReturn(rc, rc);
11989 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11990
11991 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11992 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11993 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11994
11995 if (RT_SUCCESS(rc))
11996 {
11997 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11998
11999 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12000 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12001 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12002 {
12003 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12004 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12005 EMInterpretWrmsr() changes it. */
12006 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12007 }
12008 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12009 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12010 else if (pMixedCtx->ecx == MSR_K6_EFER)
12011 {
12012 /*
12013 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12014 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12015 * the other bits as well, SCE and NXE. See @bugref{7368}.
12016 */
12017 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12018 }
12019
12020 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12021 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12022 {
12023 switch (pMixedCtx->ecx)
12024 {
12025 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12026 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12027 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12028 case MSR_K8_FS_BASE: /* no break */
12029 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12030 case MSR_K6_EFER: /* already handled above */ break;
12031 default:
12032 {
12033 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12034 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12035 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12036 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12037 break;
12038 }
12039 }
12040 }
12041#ifdef VBOX_STRICT
12042 else
12043 {
12044 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12045 switch (pMixedCtx->ecx)
12046 {
12047 case MSR_IA32_SYSENTER_CS:
12048 case MSR_IA32_SYSENTER_EIP:
12049 case MSR_IA32_SYSENTER_ESP:
12050 case MSR_K8_FS_BASE:
12051 case MSR_K8_GS_BASE:
12052 {
12053 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12054 HMVMX_RETURN_UNEXPECTED_EXIT();
12055 }
12056
12057 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12058 default:
12059 {
12060 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12061 {
12062 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12063 if (pMixedCtx->ecx != MSR_K6_EFER)
12064 {
12065 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12066 pMixedCtx->ecx));
12067 HMVMX_RETURN_UNEXPECTED_EXIT();
12068 }
12069 }
12070
12071 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12072 {
12073 VMXMSREXITREAD enmRead;
12074 VMXMSREXITWRITE enmWrite;
12075 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12076 AssertRCReturn(rc2, rc2);
12077 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12078 {
12079 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12080 HMVMX_RETURN_UNEXPECTED_EXIT();
12081 }
12082 }
12083 break;
12084 }
12085 }
12086 }
12087#endif /* VBOX_STRICT */
12088 }
12089 return rc;
12090}
12091
12092
12093/**
12094 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12095 */
12096HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12097{
12098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12099
12100 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12101 return VINF_EM_RAW_INTERRUPT;
12102}
12103
12104
12105/**
12106 * VM-exit handler for when the TPR value is lowered below the specified
12107 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12108 */
12109HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12110{
12111 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12112 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12113
12114 /*
12115 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12116 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12117 * resume guest execution.
12118 */
12119 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12121 return VINF_SUCCESS;
12122}
12123
12124
12125/**
12126 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12127 * VM-exit.
12128 *
12129 * @retval VINF_SUCCESS when guest execution can continue.
12130 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12131 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12132 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12133 * interpreter.
12134 */
12135HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12136{
12137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12138 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12139 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12140 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12141 AssertRCReturn(rc, rc);
12142
12143 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12144 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12145 PVM pVM = pVCpu->CTX_SUFF(pVM);
12146 VBOXSTRICTRC rcStrict;
12147 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12148 switch (uAccessType)
12149 {
12150 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12151 {
12152 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12153 AssertRCReturn(rc, rc);
12154
12155 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12156 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12157 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12158 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12159 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12160 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12161 {
12162 case 0: /* CR0 */
12163 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12164 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12165 break;
12166 case 2: /* CR2 */
12167 /* Nothing to do here, CR2 it's not part of the VMCS. */
12168 break;
12169 case 3: /* CR3 */
12170 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12171 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12172 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12173 break;
12174 case 4: /* CR4 */
12175 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12176 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12177 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12178 break;
12179 case 8: /* CR8 */
12180 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12181 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12182 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12183 break;
12184 default:
12185 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12186 break;
12187 }
12188
12189 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12190 break;
12191 }
12192
12193 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12194 {
12195 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12196 AssertRCReturn(rc, rc);
12197
12198 Assert( !pVM->hm.s.fNestedPaging
12199 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12200 || pVCpu->hm.s.fUsingDebugLoop
12201 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12202
12203 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12204 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12205 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12206
12207 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12208 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12209 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12210 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12211 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12212 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12213 VBOXSTRICTRC_VAL(rcStrict)));
12214 break;
12215 }
12216
12217 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12218 {
12219 AssertRCReturn(rc, rc);
12220 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12221 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12222 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12223 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12224 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12225 break;
12226 }
12227
12228 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12229 {
12230 AssertRCReturn(rc, rc);
12231 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12232 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12233 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12234 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12236 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12237 break;
12238 }
12239
12240 default:
12241 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12242 VERR_VMX_UNEXPECTED_EXCEPTION);
12243 }
12244
12245 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12246 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12247 NOREF(pVM);
12248 return rcStrict;
12249}
12250
12251
12252/**
12253 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12254 * VM-exit.
12255 */
12256HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12257{
12258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12259 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12260
12261 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12262 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12263 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12264 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12265 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12266 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12267 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12268 AssertRCReturn(rc2, rc2);
12269
12270 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12271 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12272 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12273 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12274 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12275 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12276 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12277 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12278 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12279
12280 /* I/O operation lookup arrays. */
12281 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12282 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12283
12284 VBOXSTRICTRC rcStrict;
12285 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12286 uint32_t const cbInstr = pVmxTransient->cbInstr;
12287 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12288 PVM pVM = pVCpu->CTX_SUFF(pVM);
12289 if (fIOString)
12290 {
12291#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12292 See @bugref{5752#c158}. Should work now. */
12293 /*
12294 * INS/OUTS - I/O String instruction.
12295 *
12296 * Use instruction-information if available, otherwise fall back on
12297 * interpreting the instruction.
12298 */
12299 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12300 fIOWrite ? 'w' : 'r'));
12301 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12302 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12303 {
12304 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12305 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12306 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12307 AssertRCReturn(rc2, rc2);
12308 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12309 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12310 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12311 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12312 if (fIOWrite)
12313 {
12314 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12315 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12316 }
12317 else
12318 {
12319 /*
12320 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12321 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12322 * See Intel Instruction spec. for "INS".
12323 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12324 */
12325 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12326 }
12327 }
12328 else
12329 {
12330 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12331 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12332 AssertRCReturn(rc2, rc2);
12333 rcStrict = IEMExecOne(pVCpu);
12334 }
12335 /** @todo IEM needs to be setting these flags somehow. */
12336 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12337 fUpdateRipAlready = true;
12338#else
12339 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12340 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12341 if (RT_SUCCESS(rcStrict))
12342 {
12343 if (fIOWrite)
12344 {
12345 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12346 (DISCPUMODE)pDis->uAddrMode, cbValue);
12347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12348 }
12349 else
12350 {
12351 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12352 (DISCPUMODE)pDis->uAddrMode, cbValue);
12353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12354 }
12355 }
12356 else
12357 {
12358 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12359 pMixedCtx->rip));
12360 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12361 }
12362#endif
12363 }
12364 else
12365 {
12366 /*
12367 * IN/OUT - I/O instruction.
12368 */
12369 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12370 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12371 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12372 if (fIOWrite)
12373 {
12374 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12375 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12376 }
12377 else
12378 {
12379 uint32_t u32Result = 0;
12380 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12381 if (IOM_SUCCESS(rcStrict))
12382 {
12383 /* Save result of I/O IN instr. in AL/AX/EAX. */
12384 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12385 }
12386 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12387 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12389 }
12390 }
12391
12392 if (IOM_SUCCESS(rcStrict))
12393 {
12394 if (!fUpdateRipAlready)
12395 {
12396 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12397 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12398 }
12399
12400 /*
12401 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12402 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12403 */
12404 if (fIOString)
12405 {
12406 /** @todo Single-step for INS/OUTS with REP prefix? */
12407 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12408 }
12409 else if ( !fDbgStepping
12410 && fGstStepping)
12411 {
12412 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12413 }
12414
12415 /*
12416 * If any I/O breakpoints are armed, we need to check if one triggered
12417 * and take appropriate action.
12418 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12419 */
12420 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12421 AssertRCReturn(rc2, rc2);
12422
12423 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12424 * execution engines about whether hyper BPs and such are pending. */
12425 uint32_t const uDr7 = pMixedCtx->dr[7];
12426 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12427 && X86_DR7_ANY_RW_IO(uDr7)
12428 && (pMixedCtx->cr4 & X86_CR4_DE))
12429 || DBGFBpIsHwIoArmed(pVM)))
12430 {
12431 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12432
12433 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12434 VMMRZCallRing3Disable(pVCpu);
12435 HM_DISABLE_PREEMPT();
12436
12437 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12438
12439 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12440 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12441 {
12442 /* Raise #DB. */
12443 if (fIsGuestDbgActive)
12444 ASMSetDR6(pMixedCtx->dr[6]);
12445 if (pMixedCtx->dr[7] != uDr7)
12446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12447
12448 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12449 }
12450 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12451 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12452 else if ( rcStrict2 != VINF_SUCCESS
12453 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12454 rcStrict = rcStrict2;
12455 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12456
12457 HM_RESTORE_PREEMPT();
12458 VMMRZCallRing3Enable(pVCpu);
12459 }
12460 }
12461
12462#ifdef VBOX_STRICT
12463 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12464 Assert(!fIOWrite);
12465 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12466 Assert(fIOWrite);
12467 else
12468 {
12469#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12470 * statuses, that the VMM device and some others may return. See
12471 * IOM_SUCCESS() for guidance. */
12472 AssertMsg( RT_FAILURE(rcStrict)
12473 || rcStrict == VINF_SUCCESS
12474 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12475 || rcStrict == VINF_EM_DBG_BREAKPOINT
12476 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12477 || rcStrict == VINF_EM_RAW_TO_R3
12478 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12479#endif
12480 }
12481#endif
12482
12483 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12484 return rcStrict;
12485}
12486
12487
12488/**
12489 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12490 * VM-exit.
12491 */
12492HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12493{
12494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12495
12496 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12497 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12498 AssertRCReturn(rc, rc);
12499 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12500 {
12501 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12502 AssertRCReturn(rc, rc);
12503 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12504 {
12505 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12506
12507 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12508 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12509
12510 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12511 Assert(!pVCpu->hm.s.Event.fPending);
12512 pVCpu->hm.s.Event.fPending = true;
12513 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12514 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12515 AssertRCReturn(rc, rc);
12516 if (fErrorCodeValid)
12517 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12518 else
12519 pVCpu->hm.s.Event.u32ErrCode = 0;
12520 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12521 && uVector == X86_XCPT_PF)
12522 {
12523 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12524 }
12525
12526 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12528 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12529 }
12530 }
12531
12532 /* Fall back to the interpreter to emulate the task-switch. */
12533 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12534 return VERR_EM_INTERPRETER;
12535}
12536
12537
12538/**
12539 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12540 */
12541HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12542{
12543 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12544 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12545 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12547 AssertRCReturn(rc, rc);
12548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12549 return VINF_EM_DBG_STEPPED;
12550}
12551
12552
12553/**
12554 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12555 */
12556HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12557{
12558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12559
12560 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12561
12562 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12563 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12564 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12565 {
12566 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12567 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12568 {
12569 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12570 return VERR_EM_INTERPRETER;
12571 }
12572 }
12573 else
12574 {
12575 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12576 rcStrict1 = VINF_SUCCESS;
12577 return rcStrict1;
12578 }
12579
12580#if 0
12581 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12582 * just sync the whole thing. */
12583 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12584#else
12585 /* Aggressive state sync. for now. */
12586 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12587 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12588 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12589#endif
12590 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12591 AssertRCReturn(rc, rc);
12592
12593 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12594 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12595 VBOXSTRICTRC rcStrict2;
12596 switch (uAccessType)
12597 {
12598 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12599 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12600 {
12601 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12602 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12603 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12604
12605 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12606 GCPhys &= PAGE_BASE_GC_MASK;
12607 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12608 PVM pVM = pVCpu->CTX_SUFF(pVM);
12609 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12610 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12611
12612 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12613 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12614 CPUMCTX2CORE(pMixedCtx), GCPhys);
12615 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12616 if ( rcStrict2 == VINF_SUCCESS
12617 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12618 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12619 {
12620 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12621 | HM_CHANGED_GUEST_RSP
12622 | HM_CHANGED_GUEST_RFLAGS
12623 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12624 rcStrict2 = VINF_SUCCESS;
12625 }
12626 break;
12627 }
12628
12629 default:
12630 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12631 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12632 break;
12633 }
12634
12635 if (rcStrict2 != VINF_SUCCESS)
12636 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12637 return rcStrict2;
12638}
12639
12640
12641/**
12642 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12643 * VM-exit.
12644 */
12645HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12646{
12647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12648
12649 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12650 if (pVmxTransient->fWasGuestDebugStateActive)
12651 {
12652 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12653 HMVMX_RETURN_UNEXPECTED_EXIT();
12654 }
12655
12656 if ( !pVCpu->hm.s.fSingleInstruction
12657 && !pVmxTransient->fWasHyperDebugStateActive)
12658 {
12659 Assert(!DBGFIsStepping(pVCpu));
12660 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12661
12662 /* Don't intercept MOV DRx any more. */
12663 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12664 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12665 AssertRCReturn(rc, rc);
12666
12667 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12668 VMMRZCallRing3Disable(pVCpu);
12669 HM_DISABLE_PREEMPT();
12670
12671 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12672 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12673 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12674
12675 HM_RESTORE_PREEMPT();
12676 VMMRZCallRing3Enable(pVCpu);
12677
12678#ifdef VBOX_WITH_STATISTICS
12679 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12680 AssertRCReturn(rc, rc);
12681 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12683 else
12684 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12685#endif
12686 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12687 return VINF_SUCCESS;
12688 }
12689
12690 /*
12691 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12692 * Update the segment registers and DR7 from the CPU.
12693 */
12694 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12695 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12696 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12697 AssertRCReturn(rc, rc);
12698 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12699
12700 PVM pVM = pVCpu->CTX_SUFF(pVM);
12701 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12702 {
12703 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12704 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12705 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12706 if (RT_SUCCESS(rc))
12707 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12708 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12709 }
12710 else
12711 {
12712 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12713 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12714 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12715 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12716 }
12717
12718 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12719 if (RT_SUCCESS(rc))
12720 {
12721 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12722 AssertRCReturn(rc2, rc2);
12723 return VINF_SUCCESS;
12724 }
12725 return rc;
12726}
12727
12728
12729/**
12730 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12731 * Conditional VM-exit.
12732 */
12733HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12734{
12735 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12736 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12737
12738 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12739 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12740 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12741 {
12742 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12743 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12744 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12745 {
12746 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12747 return VERR_EM_INTERPRETER;
12748 }
12749 }
12750 else
12751 {
12752 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12753 rcStrict1 = VINF_SUCCESS;
12754 return rcStrict1;
12755 }
12756
12757 RTGCPHYS GCPhys = 0;
12758 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12759
12760#if 0
12761 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12762#else
12763 /* Aggressive state sync. for now. */
12764 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12765 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12766 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12767#endif
12768 AssertRCReturn(rc, rc);
12769
12770 /*
12771 * If we succeed, resume guest execution.
12772 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12773 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12774 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12775 * weird case. See @bugref{6043}.
12776 */
12777 PVM pVM = pVCpu->CTX_SUFF(pVM);
12778 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12779 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12780 if ( rcStrict2 == VINF_SUCCESS
12781 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12782 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12783 {
12784 /* Successfully handled MMIO operation. */
12785 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12786 | HM_CHANGED_GUEST_RSP
12787 | HM_CHANGED_GUEST_RFLAGS
12788 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12789 return VINF_SUCCESS;
12790 }
12791 return rcStrict2;
12792}
12793
12794
12795/**
12796 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12797 * VM-exit.
12798 */
12799HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12800{
12801 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12802 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12803
12804 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12805 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12806 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12807 {
12808 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12809 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12810 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12811 }
12812 else
12813 {
12814 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12815 rcStrict1 = VINF_SUCCESS;
12816 return rcStrict1;
12817 }
12818
12819 RTGCPHYS GCPhys = 0;
12820 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12821 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12822#if 0
12823 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12824#else
12825 /* Aggressive state sync. for now. */
12826 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12827 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12828 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12829#endif
12830 AssertRCReturn(rc, rc);
12831
12832 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12833 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12834
12835 RTGCUINT uErrorCode = 0;
12836 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12837 uErrorCode |= X86_TRAP_PF_ID;
12838 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12839 uErrorCode |= X86_TRAP_PF_RW;
12840 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12841 uErrorCode |= X86_TRAP_PF_P;
12842
12843 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12844
12845 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12846 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12847
12848 /* Handle the pagefault trap for the nested shadow table. */
12849 PVM pVM = pVCpu->CTX_SUFF(pVM);
12850 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12851 TRPMResetTrap(pVCpu);
12852
12853 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12854 if ( rcStrict2 == VINF_SUCCESS
12855 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12856 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12857 {
12858 /* Successfully synced our nested page tables. */
12859 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12860 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12861 | HM_CHANGED_GUEST_RSP
12862 | HM_CHANGED_GUEST_RFLAGS);
12863 return VINF_SUCCESS;
12864 }
12865
12866 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12867 return rcStrict2;
12868}
12869
12870/** @} */
12871
12872/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12873/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12874/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12875
12876/** @name VM-exit exception handlers.
12877 * @{
12878 */
12879
12880/**
12881 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12882 */
12883static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12884{
12885 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12887
12888 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12889 AssertRCReturn(rc, rc);
12890
12891 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12892 {
12893 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12894 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12895
12896 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12897 * provides VM-exit instruction length. If this causes problem later,
12898 * disassemble the instruction like it's done on AMD-V. */
12899 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12900 AssertRCReturn(rc2, rc2);
12901 return rc;
12902 }
12903
12904 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12905 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12906 return rc;
12907}
12908
12909
12910/**
12911 * VM-exit exception handler for \#BP (Breakpoint exception).
12912 */
12913static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12914{
12915 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12917
12918 /** @todo Try optimize this by not saving the entire guest state unless
12919 * really needed. */
12920 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12921 AssertRCReturn(rc, rc);
12922
12923 PVM pVM = pVCpu->CTX_SUFF(pVM);
12924 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12925 if (rc == VINF_EM_RAW_GUEST_TRAP)
12926 {
12927 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12928 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12929 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12930 AssertRCReturn(rc, rc);
12931
12932 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12933 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12934 }
12935
12936 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12937 return rc;
12938}
12939
12940
12941/**
12942 * VM-exit exception handler for \#AC (alignment check exception).
12943 */
12944static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12945{
12946 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12947
12948 /*
12949 * Re-inject it. We'll detect any nesting before getting here.
12950 */
12951 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12952 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12953 AssertRCReturn(rc, rc);
12954 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12955
12956 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12957 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12958 return VINF_SUCCESS;
12959}
12960
12961
12962/**
12963 * VM-exit exception handler for \#DB (Debug exception).
12964 */
12965static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12966{
12967 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12968 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12969 Log6(("XcptDB\n"));
12970
12971 /*
12972 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12973 * for processing.
12974 */
12975 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12976 AssertRCReturn(rc, rc);
12977
12978 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12979 uint64_t uDR6 = X86_DR6_INIT_VAL;
12980 uDR6 |= ( pVmxTransient->uExitQualification
12981 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12982
12983 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12984 if (rc == VINF_EM_RAW_GUEST_TRAP)
12985 {
12986 /*
12987 * The exception was for the guest. Update DR6, DR7.GD and
12988 * IA32_DEBUGCTL.LBR before forwarding it.
12989 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12990 */
12991 VMMRZCallRing3Disable(pVCpu);
12992 HM_DISABLE_PREEMPT();
12993
12994 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12995 pMixedCtx->dr[6] |= uDR6;
12996 if (CPUMIsGuestDebugStateActive(pVCpu))
12997 ASMSetDR6(pMixedCtx->dr[6]);
12998
12999 HM_RESTORE_PREEMPT();
13000 VMMRZCallRing3Enable(pVCpu);
13001
13002 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13003 AssertRCReturn(rc, rc);
13004
13005 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13006 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13007
13008 /* Paranoia. */
13009 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13010 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13011
13012 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13013 AssertRCReturn(rc, rc);
13014
13015 /*
13016 * Raise #DB in the guest.
13017 *
13018 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13019 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13020 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13021 *
13022 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13023 */
13024 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13025 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13026 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13027 AssertRCReturn(rc, rc);
13028 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13029 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13030 return VINF_SUCCESS;
13031 }
13032
13033 /*
13034 * Not a guest trap, must be a hypervisor related debug event then.
13035 * Update DR6 in case someone is interested in it.
13036 */
13037 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13038 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13039 CPUMSetHyperDR6(pVCpu, uDR6);
13040
13041 return rc;
13042}
13043
13044
13045/**
13046 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13047 * point exception).
13048 */
13049static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13050{
13051 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13052
13053 /* We require CR0 and EFER. EFER is always up-to-date. */
13054 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13055 AssertRCReturn(rc, rc);
13056
13057 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13058 VMMRZCallRing3Disable(pVCpu);
13059 HM_DISABLE_PREEMPT();
13060
13061 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13062 if (pVmxTransient->fWasGuestFPUStateActive)
13063 {
13064 rc = VINF_EM_RAW_GUEST_TRAP;
13065 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13066 }
13067 else
13068 {
13069#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13070 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13071#endif
13072 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13073 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13074 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13075 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13076 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13077 }
13078
13079 HM_RESTORE_PREEMPT();
13080 VMMRZCallRing3Enable(pVCpu);
13081
13082 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13083 {
13084 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13085 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13087 pVCpu->hm.s.fPreloadGuestFpu = true;
13088 }
13089 else
13090 {
13091 /* Forward #NM to the guest. */
13092 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13093 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13094 AssertRCReturn(rc, rc);
13095 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13096 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13097 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13098 }
13099
13100 return VINF_SUCCESS;
13101}
13102
13103
13104/**
13105 * VM-exit exception handler for \#GP (General-protection exception).
13106 *
13107 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13108 */
13109static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13110{
13111 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13113
13114 int rc;
13115 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13116 { /* likely */ }
13117 else
13118 {
13119#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13120 Assert(pVCpu->hm.s.fUsingDebugLoop);
13121#endif
13122 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13123 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13124 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13125 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13126 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13127 AssertRCReturn(rc, rc);
13128 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13129 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13130 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13131 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13132 return rc;
13133 }
13134
13135 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13136 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13137
13138 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13139 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13140 AssertRCReturn(rc, rc);
13141
13142 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13143 uint32_t cbOp = 0;
13144 PVM pVM = pVCpu->CTX_SUFF(pVM);
13145 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13146 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13147 if (RT_SUCCESS(rc))
13148 {
13149 rc = VINF_SUCCESS;
13150 Assert(cbOp == pDis->cbInstr);
13151 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13152 switch (pDis->pCurInstr->uOpcode)
13153 {
13154 case OP_CLI:
13155 {
13156 pMixedCtx->eflags.Bits.u1IF = 0;
13157 pMixedCtx->eflags.Bits.u1RF = 0;
13158 pMixedCtx->rip += pDis->cbInstr;
13159 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13160 if ( !fDbgStepping
13161 && pMixedCtx->eflags.Bits.u1TF)
13162 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13163 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13164 break;
13165 }
13166
13167 case OP_STI:
13168 {
13169 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13170 pMixedCtx->eflags.Bits.u1IF = 1;
13171 pMixedCtx->eflags.Bits.u1RF = 0;
13172 pMixedCtx->rip += pDis->cbInstr;
13173 if (!fOldIF)
13174 {
13175 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13176 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13177 }
13178 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13179 if ( !fDbgStepping
13180 && pMixedCtx->eflags.Bits.u1TF)
13181 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13182 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13183 break;
13184 }
13185
13186 case OP_HLT:
13187 {
13188 rc = VINF_EM_HALT;
13189 pMixedCtx->rip += pDis->cbInstr;
13190 pMixedCtx->eflags.Bits.u1RF = 0;
13191 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13193 break;
13194 }
13195
13196 case OP_POPF:
13197 {
13198 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13199 uint32_t cbParm;
13200 uint32_t uMask;
13201 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13202 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13203 {
13204 cbParm = 4;
13205 uMask = 0xffffffff;
13206 }
13207 else
13208 {
13209 cbParm = 2;
13210 uMask = 0xffff;
13211 }
13212
13213 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13214 RTGCPTR GCPtrStack = 0;
13215 X86EFLAGS Eflags;
13216 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13217 &GCPtrStack);
13218 if (RT_SUCCESS(rc))
13219 {
13220 Assert(sizeof(Eflags.u32) >= cbParm);
13221 Eflags.u32 = 0;
13222 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13223 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13224 }
13225 if (RT_FAILURE(rc))
13226 {
13227 rc = VERR_EM_INTERPRETER;
13228 break;
13229 }
13230 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13231 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13232 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13233 pMixedCtx->esp += cbParm;
13234 pMixedCtx->esp &= uMask;
13235 pMixedCtx->rip += pDis->cbInstr;
13236 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13237 | HM_CHANGED_GUEST_RSP
13238 | HM_CHANGED_GUEST_RFLAGS);
13239 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13240 POPF restores EFLAGS.TF. */
13241 if ( !fDbgStepping
13242 && fGstStepping)
13243 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13244 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13245 break;
13246 }
13247
13248 case OP_PUSHF:
13249 {
13250 uint32_t cbParm;
13251 uint32_t uMask;
13252 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13253 {
13254 cbParm = 4;
13255 uMask = 0xffffffff;
13256 }
13257 else
13258 {
13259 cbParm = 2;
13260 uMask = 0xffff;
13261 }
13262
13263 /* Get the stack pointer & push the contents of eflags onto the stack. */
13264 RTGCPTR GCPtrStack = 0;
13265 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13266 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13267 if (RT_FAILURE(rc))
13268 {
13269 rc = VERR_EM_INTERPRETER;
13270 break;
13271 }
13272 X86EFLAGS Eflags = pMixedCtx->eflags;
13273 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13274 Eflags.Bits.u1RF = 0;
13275 Eflags.Bits.u1VM = 0;
13276
13277 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13278 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13279 {
13280 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13281 rc = VERR_EM_INTERPRETER;
13282 break;
13283 }
13284 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13285 pMixedCtx->esp -= cbParm;
13286 pMixedCtx->esp &= uMask;
13287 pMixedCtx->rip += pDis->cbInstr;
13288 pMixedCtx->eflags.Bits.u1RF = 0;
13289 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13290 | HM_CHANGED_GUEST_RSP
13291 | HM_CHANGED_GUEST_RFLAGS);
13292 if ( !fDbgStepping
13293 && pMixedCtx->eflags.Bits.u1TF)
13294 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13296 break;
13297 }
13298
13299 case OP_IRET:
13300 {
13301 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13302 * instruction reference. */
13303 RTGCPTR GCPtrStack = 0;
13304 uint32_t uMask = 0xffff;
13305 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13306 uint16_t aIretFrame[3];
13307 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13308 {
13309 rc = VERR_EM_INTERPRETER;
13310 break;
13311 }
13312 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13313 &GCPtrStack);
13314 if (RT_SUCCESS(rc))
13315 {
13316 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13317 PGMACCESSORIGIN_HM));
13318 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13319 }
13320 if (RT_FAILURE(rc))
13321 {
13322 rc = VERR_EM_INTERPRETER;
13323 break;
13324 }
13325 pMixedCtx->eip = 0;
13326 pMixedCtx->ip = aIretFrame[0];
13327 pMixedCtx->cs.Sel = aIretFrame[1];
13328 pMixedCtx->cs.ValidSel = aIretFrame[1];
13329 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13330 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13331 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13332 pMixedCtx->sp += sizeof(aIretFrame);
13333 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13334 | HM_CHANGED_GUEST_SEGMENT_REGS
13335 | HM_CHANGED_GUEST_RSP
13336 | HM_CHANGED_GUEST_RFLAGS);
13337 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13338 if ( !fDbgStepping
13339 && fGstStepping)
13340 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13341 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13342 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13343 break;
13344 }
13345
13346 case OP_INT:
13347 {
13348 uint16_t uVector = pDis->Param1.uValue & 0xff;
13349 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13350 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13352 break;
13353 }
13354
13355 case OP_INTO:
13356 {
13357 if (pMixedCtx->eflags.Bits.u1OF)
13358 {
13359 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13360 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13361 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13362 }
13363 else
13364 {
13365 pMixedCtx->eflags.Bits.u1RF = 0;
13366 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13367 }
13368 break;
13369 }
13370
13371 default:
13372 {
13373 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13374 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13375 EMCODETYPE_SUPERVISOR);
13376 rc = VBOXSTRICTRC_VAL(rc2);
13377 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13378 /** @todo We have to set pending-debug exceptions here when the guest is
13379 * single-stepping depending on the instruction that was interpreted. */
13380 Log4(("#GP rc=%Rrc\n", rc));
13381 break;
13382 }
13383 }
13384 }
13385 else
13386 rc = VERR_EM_INTERPRETER;
13387
13388 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13389 ("#GP Unexpected rc=%Rrc\n", rc));
13390 return rc;
13391}
13392
13393
13394/**
13395 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13396 * the exception reported in the VMX transient structure back into the VM.
13397 *
13398 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13399 * up-to-date.
13400 */
13401static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13402{
13403 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13404#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13405 Assert(pVCpu->hm.s.fUsingDebugLoop);
13406#endif
13407
13408 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13409 hmR0VmxCheckExitDueToEventDelivery(). */
13410 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13411 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13412 AssertRCReturn(rc, rc);
13413 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13414
13415#ifdef DEBUG_ramshankar
13416 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13417 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13418 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13419#endif
13420
13421 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13422 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13423 return VINF_SUCCESS;
13424}
13425
13426
13427/**
13428 * VM-exit exception handler for \#PF (Page-fault exception).
13429 */
13430static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13431{
13432 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13433 PVM pVM = pVCpu->CTX_SUFF(pVM);
13434 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13435 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13436 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13437 AssertRCReturn(rc, rc);
13438
13439 if (!pVM->hm.s.fNestedPaging)
13440 { /* likely */ }
13441 else
13442 {
13443#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13444 Assert(pVCpu->hm.s.fUsingDebugLoop);
13445#endif
13446 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13447 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13448 {
13449 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13450 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13451 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13452 }
13453 else
13454 {
13455 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13456 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13457 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13458 }
13459 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13460 return rc;
13461 }
13462
13463 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13464 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13465 if (pVmxTransient->fVectoringPF)
13466 {
13467 Assert(pVCpu->hm.s.Event.fPending);
13468 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13469 }
13470
13471 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13472 AssertRCReturn(rc, rc);
13473
13474 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13475 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13476
13477 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13478 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13479 (RTGCPTR)pVmxTransient->uExitQualification);
13480
13481 Log4(("#PF: rc=%Rrc\n", rc));
13482 if (rc == VINF_SUCCESS)
13483 {
13484#if 0
13485 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13486 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13487 * memory? We don't update the whole state here... */
13488 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13489 | HM_CHANGED_GUEST_RSP
13490 | HM_CHANGED_GUEST_RFLAGS
13491 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13492#else
13493 /*
13494 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13495 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13496 */
13497 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13498 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13499#endif
13500 TRPMResetTrap(pVCpu);
13501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13502 return rc;
13503 }
13504
13505 if (rc == VINF_EM_RAW_GUEST_TRAP)
13506 {
13507 if (!pVmxTransient->fVectoringDoublePF)
13508 {
13509 /* It's a guest page fault and needs to be reflected to the guest. */
13510 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13511 TRPMResetTrap(pVCpu);
13512 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13513 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13514 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13515 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13516 }
13517 else
13518 {
13519 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13520 TRPMResetTrap(pVCpu);
13521 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13522 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13523 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13524 }
13525
13526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13527 return VINF_SUCCESS;
13528 }
13529
13530 TRPMResetTrap(pVCpu);
13531 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13532 return rc;
13533}
13534
13535/** @} */
13536
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette