VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 61977

Last change on this file since 61977 was 61977, checked in by vboxsync, 9 years ago

VMX: Allow 32-bit switcher again if V86 on VMX is in effect.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 580.1 KB
Line 
1/* $Id: HMVMXR0.cpp 61977 2016-07-01 12:27:46Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 *
2912 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2913 * Was observed booting Solaris10u10 32-bit guest.
2914 */
2915 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2916 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2917 {
2918 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2919 pVCpu->idCpu));
2920 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2921 }
2922 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2923#endif
2924
2925 /*
2926 * Host DS, ES, FS and GS segment registers.
2927 */
2928#if HC_ARCH_BITS == 64
2929 RTSEL uSelDS = ASMGetDS();
2930 RTSEL uSelES = ASMGetES();
2931 RTSEL uSelFS = ASMGetFS();
2932 RTSEL uSelGS = ASMGetGS();
2933#else
2934 RTSEL uSelDS = 0;
2935 RTSEL uSelES = 0;
2936 RTSEL uSelFS = 0;
2937 RTSEL uSelGS = 0;
2938#endif
2939
2940 /*
2941 * Host CS and SS segment registers.
2942 */
2943 RTSEL uSelCS = ASMGetCS();
2944 RTSEL uSelSS = ASMGetSS();
2945
2946 /*
2947 * Host TR segment register.
2948 */
2949 RTSEL uSelTR = ASMGetTR();
2950
2951#if HC_ARCH_BITS == 64
2952 /*
2953 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2954 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2955 */
2956 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2957 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2958 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2959 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2960# undef VMXLOCAL_ADJUST_HOST_SEG
2961#endif
2962
2963 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2964 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2965 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2966 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2967 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2968 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2969 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2970 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2971 Assert(uSelCS);
2972 Assert(uSelTR);
2973
2974 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2975#if 0
2976 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2977 Assert(uSelSS != 0);
2978#endif
2979
2980 /* Write these host selector fields into the host-state area in the VMCS. */
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2983#if HC_ARCH_BITS == 64
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2986 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2987 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2988#else
2989 NOREF(uSelDS);
2990 NOREF(uSelES);
2991 NOREF(uSelFS);
2992 NOREF(uSelGS);
2993#endif
2994 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2995 AssertRCReturn(rc, rc);
2996
2997 /*
2998 * Host GDTR and IDTR.
2999 */
3000 RTGDTR Gdtr;
3001 RTIDTR Idtr;
3002 RT_ZERO(Gdtr);
3003 RT_ZERO(Idtr);
3004 ASMGetGDTR(&Gdtr);
3005 ASMGetIDTR(&Idtr);
3006 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3007 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3008 AssertRCReturn(rc, rc);
3009
3010#if HC_ARCH_BITS == 64
3011 /*
3012 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3013 * maximum limit (0xffff) on every VM-exit.
3014 */
3015 if (Gdtr.cbGdt != 0xffff)
3016 {
3017 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3018 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3019 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3020 }
3021
3022 /*
3023 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3024 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3025 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3026 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3027 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3028 * hosts where we are pretty sure it won't cause trouble.
3029 */
3030# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3031 if (Idtr.cbIdt < 0x0fff)
3032# else
3033 if (Idtr.cbIdt != 0xffff)
3034# endif
3035 {
3036 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3037 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3038 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3039 }
3040#endif
3041
3042 /*
3043 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3044 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3045 */
3046 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3047 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3048 VERR_VMX_INVALID_HOST_STATE);
3049
3050 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3051#if HC_ARCH_BITS == 64
3052 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3053
3054 /*
3055 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3056 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3057 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3058 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3059 *
3060 * [1] See Intel spec. 3.5 "System Descriptor Types".
3061 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3062 */
3063 Assert(pDesc->System.u4Type == 11);
3064 if ( pDesc->System.u16LimitLow != 0x67
3065 || pDesc->System.u4LimitHigh)
3066 {
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3068 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3069 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3071 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3072
3073 /* Store the GDTR here as we need it while restoring TR. */
3074 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3075 }
3076#else
3077 NOREF(pVM);
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM The cross context VM structure.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 /*
3122 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3123 * rather than swapping them on every VM-entry.
3124 */
3125 hmR0VmxLazySaveHostMsrs(pVCpu);
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131#if HC_ARCH_BITS == 32
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3133 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3134#else
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3137#endif
3138 AssertRCReturn(rc, rc);
3139
3140 /*
3141 * Host EFER MSR.
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3143 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3144 */
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3152 * hmR0VmxLoadGuestExitCtls() !! */
3153
3154 return rc;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3163 * hmR0VMxLoadGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3168 * out-of-sync. Make sure to update the required fields
3169 * before using them.
3170 *
3171 * @remarks Requires EFER, CR4.
3172 * @remarks No-long-jump zone!!!
3173 */
3174static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3175{
3176#ifdef HMVMX_ALWAYS_SWAP_EFER
3177 return true;
3178#endif
3179
3180#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3181 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3182 if (CPUMIsGuestInLongMode(pVCpu))
3183 return false;
3184#endif
3185
3186 PVM pVM = pVCpu->CTX_SUFF(pVM);
3187 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3188 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3189
3190 /*
3191 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3192 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3193 */
3194 if ( CPUMIsGuestInLongMode(pVCpu)
3195 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3196 {
3197 return true;
3198 }
3199
3200 /*
3201 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3202 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3203 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3204 */
3205 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3206 && (pMixedCtx->cr0 & X86_CR0_PG)
3207 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3208 {
3209 /* Assert that host is PAE capable. */
3210 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3211 return true;
3212 }
3213
3214 /** @todo Check the latest Intel spec. for any other bits,
3215 * like SMEP/SMAP? */
3216 return false;
3217}
3218
3219
3220/**
3221 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3222 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3223 * controls".
3224 *
3225 * @returns VBox status code.
3226 * @param pVCpu The cross context virtual CPU structure.
3227 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3228 * out-of-sync. Make sure to update the required fields
3229 * before using them.
3230 *
3231 * @remarks Requires EFER.
3232 * @remarks No-long-jump zone!!!
3233 */
3234DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3235{
3236 int rc = VINF_SUCCESS;
3237 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3238 {
3239 PVM pVM = pVCpu->CTX_SUFF(pVM);
3240 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3241 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3242
3243 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3244 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3245
3246 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3247 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3248 {
3249 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3250 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3251 }
3252 else
3253 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3254
3255 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3256 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3257 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3258 {
3259 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3260 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3261 }
3262
3263 /*
3264 * The following should -not- be set (since we're not in SMM mode):
3265 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3266 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3267 */
3268
3269 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3270 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3271
3272 if ((val & zap) != val)
3273 {
3274 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3275 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3276 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3277 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3278 }
3279
3280 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3281 AssertRCReturn(rc, rc);
3282
3283 pVCpu->hm.s.vmx.u32EntryCtls = val;
3284 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3285 }
3286 return rc;
3287}
3288
3289
3290/**
3291 * Sets up the VM-exit controls in the VMCS.
3292 *
3293 * @returns VBox status code.
3294 * @param pVCpu The cross context virtual CPU structure.
3295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3296 * out-of-sync. Make sure to update the required fields
3297 * before using them.
3298 *
3299 * @remarks Requires EFER.
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 NOREF(pMixedCtx);
3304
3305 int rc = VINF_SUCCESS;
3306 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3307 {
3308 PVM pVM = pVCpu->CTX_SUFF(pVM);
3309 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3310 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3311
3312 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3313 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3314
3315 /*
3316 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3317 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3318 */
3319#if HC_ARCH_BITS == 64
3320 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3322#else
3323 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3324 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3325 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3326 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3327 {
3328 /* The switcher returns to long mode, EFER is managed by the switcher. */
3329 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3330 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3331 }
3332 else
3333 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3334#endif
3335
3336 /* If the newer VMCS fields for managing EFER exists, use it. */
3337 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3338 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3339 {
3340 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3341 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3342 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3343 }
3344
3345 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3346 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3347
3348 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3349 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3350 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3351
3352 if ( pVM->hm.s.vmx.fUsePreemptTimer
3353 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3354 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3355
3356 if ((val & zap) != val)
3357 {
3358 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3359 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3360 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3361 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3362 }
3363
3364 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3365 AssertRCReturn(rc, rc);
3366
3367 pVCpu->hm.s.vmx.u32ExitCtls = val;
3368 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3369 }
3370 return rc;
3371}
3372
3373
3374/**
3375 * Sets the TPR threshold in the VMCS.
3376 *
3377 * @returns VBox status code.
3378 * @param pVCpu The cross context virtual CPU structure.
3379 * @param u32TprThreshold The TPR threshold (task-priority class only).
3380 */
3381DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3382{
3383 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3385 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3386}
3387
3388
3389/**
3390 * Loads the guest APIC and related state.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3395 * out-of-sync. Make sure to update the required fields
3396 * before using them.
3397 */
3398DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3399{
3400 NOREF(pMixedCtx);
3401
3402 int rc = VINF_SUCCESS;
3403 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3404 {
3405 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3418 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3419 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3420 * the interrupt when we VM-exit for other reasons.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3428 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 else
3432 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3433 }
3434
3435 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3436 AssertRCReturn(rc, rc);
3437 }
3438
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3447 *
3448 * @returns Guest's interruptibility-state.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3451 * out-of-sync. Make sure to update the required fields
3452 * before using them.
3453 *
3454 * @remarks No-long-jump zone!!!
3455 */
3456DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3457{
3458 /*
3459 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pMixedCtx->eflags.Bits.u1IF)
3470 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3478 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3493 {
3494 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3495 }
3496
3497 return uIntrState;
3498}
3499
3500
3501/**
3502 * Loads the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param uIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3515 AssertRC(rc);
3516 return rc;
3517}
3518
3519
3520/**
3521 * Loads the exception intercepts required for guest execution in the VMCS.
3522 *
3523 * @returns VBox status code.
3524 * @param pVCpu The cross context virtual CPU structure.
3525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3526 * out-of-sync. Make sure to update the required fields
3527 * before using them.
3528 */
3529static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3530{
3531 NOREF(pMixedCtx);
3532 int rc = VINF_SUCCESS;
3533 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3534 {
3535 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3536 if (pVCpu->hm.s.fGIMTrapXcptUD)
3537 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3538#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3539 else
3540 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3541#endif
3542
3543 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3544 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3545
3546 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3550 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3551 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3577 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3578 HMCPU_CF_VALUE(pVCpu)));
3579 }
3580 return rc;
3581}
3582
3583
3584/**
3585 * Loads the guest's RSP into the guest-state area in the VMCS.
3586 *
3587 * @returns VBox status code.
3588 * @param pVCpu The cross context virtual CPU structure.
3589 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3590 * out-of-sync. Make sure to update the required fields
3591 * before using them.
3592 *
3593 * @remarks No-long-jump zone!!!
3594 */
3595static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3596{
3597 int rc = VINF_SUCCESS;
3598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3599 {
3600 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3601 AssertRCReturn(rc, rc);
3602
3603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3604 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3605 }
3606 return rc;
3607}
3608
3609
3610/**
3611 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3612 *
3613 * @returns VBox status code.
3614 * @param pVCpu The cross context virtual CPU structure.
3615 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3616 * out-of-sync. Make sure to update the required fields
3617 * before using them.
3618 *
3619 * @remarks No-long-jump zone!!!
3620 */
3621static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3622{
3623 int rc = VINF_SUCCESS;
3624 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3625 {
3626 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3627 Let us assert it as such and use 32-bit VMWRITE. */
3628 Assert(!(pMixedCtx->rflags.u64 >> 32));
3629 X86EFLAGS Eflags = pMixedCtx->eflags;
3630 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3631 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3632 * These will never be cleared/set, unless some other part of the VMM
3633 * code is buggy - in which case we're better of finding and fixing
3634 * those bugs than hiding them. */
3635 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3636 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3637 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3638 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3639
3640 /*
3641 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3642 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3643 */
3644 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3645 {
3646 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3647 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3648 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3649 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3650 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3651 }
3652
3653 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3654 AssertRCReturn(rc, rc);
3655
3656 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3657 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3658 }
3659 return rc;
3660}
3661
3662
3663/**
3664 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3665 *
3666 * @returns VBox status code.
3667 * @param pVCpu The cross context virtual CPU structure.
3668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3669 * out-of-sync. Make sure to update the required fields
3670 * before using them.
3671 *
3672 * @remarks No-long-jump zone!!!
3673 */
3674DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3675{
3676 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3677 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3678 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3679 AssertRCReturn(rc, rc);
3680 return rc;
3681}
3682
3683
3684/**
3685 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3686 * CR0 is partially shared with the host and we have to consider the FPU bits.
3687 *
3688 * @returns VBox status code.
3689 * @param pVCpu The cross context virtual CPU structure.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 {
3777 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3778 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3779 }
3780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3781
3782 if (fInterceptNM)
3783 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3784 else
3785 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3786
3787 if (fInterceptMF)
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3789 else
3790 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3791
3792 /* Additional intercepts for debugging, define these yourself explicitly. */
3793#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3794 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3795 | RT_BIT(X86_XCPT_BP)
3796 | RT_BIT(X86_XCPT_DE)
3797 | RT_BIT(X86_XCPT_NM)
3798 | RT_BIT(X86_XCPT_TS)
3799 | RT_BIT(X86_XCPT_UD)
3800 | RT_BIT(X86_XCPT_NP)
3801 | RT_BIT(X86_XCPT_SS)
3802 | RT_BIT(X86_XCPT_GP)
3803 | RT_BIT(X86_XCPT_PF)
3804 | RT_BIT(X86_XCPT_MF)
3805 ;
3806#elif defined(HMVMX_ALWAYS_TRAP_PF)
3807 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3808#endif
3809
3810 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3811
3812 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3813 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3814 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3816 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3817 else
3818 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3819
3820 u32GuestCR0 |= uSetCR0;
3821 u32GuestCR0 &= uZapCR0;
3822 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3823
3824 /* Write VT-x's view of the guest CR0 into the VMCS. */
3825 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3826 AssertRCReturn(rc, rc);
3827 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3828 uZapCR0));
3829
3830 /*
3831 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3832 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3833 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3834 */
3835 uint32_t u32CR0Mask = 0;
3836 u32CR0Mask = X86_CR0_PE
3837 | X86_CR0_NE
3838 | X86_CR0_WP
3839 | X86_CR0_PG
3840 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3841 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3842 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3843
3844 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3845 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3846 * and @bugref{6944}. */
3847#if 0
3848 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3849 u32CR0Mask &= ~X86_CR0_PE;
3850#endif
3851 if (pVM->hm.s.fNestedPaging)
3852 u32CR0Mask &= ~X86_CR0_WP;
3853
3854 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3855 if (fInterceptNM)
3856 {
3857 u32CR0Mask |= X86_CR0_TS
3858 | X86_CR0_MP;
3859 }
3860
3861 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3862 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3863 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3864 AssertRCReturn(rc, rc);
3865 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3866
3867 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3868 }
3869 return rc;
3870}
3871
3872
3873/**
3874 * Loads the guest control registers (CR3, CR4) into the guest-state area
3875 * in the VMCS.
3876 *
3877 * @returns VBox strict status code.
3878 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3879 * without unrestricted guest access and the VMMDev is not presently
3880 * mapped (e.g. EFI32).
3881 *
3882 * @param pVCpu The cross context virtual CPU structure.
3883 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3884 * out-of-sync. Make sure to update the required fields
3885 * before using them.
3886 *
3887 * @remarks No-long-jump zone!!!
3888 */
3889static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3890{
3891 int rc = VINF_SUCCESS;
3892 PVM pVM = pVCpu->CTX_SUFF(pVM);
3893
3894 /*
3895 * Guest CR2.
3896 * It's always loaded in the assembler code. Nothing to do here.
3897 */
3898
3899 /*
3900 * Guest CR3.
3901 */
3902 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3903 {
3904 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3905 if (pVM->hm.s.fNestedPaging)
3906 {
3907 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3908
3909 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3910 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3911 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3912 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3913
3914 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3915 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3916 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3917
3918 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3919 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3920 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3921 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3922 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3923 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3924 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3925
3926 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3927 AssertRCReturn(rc, rc);
3928 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3929
3930 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3931 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3932 {
3933 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3934 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3935 {
3936 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3937 AssertRCReturn(rc, rc);
3938 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3939 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3940 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3941 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3946 have Unrestricted Execution to handle the guest when it's not using paging. */
3947 GCPhysGuestCR3 = pMixedCtx->cr3;
3948 }
3949 else
3950 {
3951 /*
3952 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3953 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3954 * EPT takes care of translating it to host-physical addresses.
3955 */
3956 RTGCPHYS GCPhys;
3957 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3958
3959 /* We obtain it here every time as the guest could have relocated this PCI region. */
3960 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3961 if (RT_SUCCESS(rc))
3962 { /* likely */ }
3963 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3964 {
3965 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3966 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3967 }
3968 else
3969 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3970
3971 GCPhysGuestCR3 = GCPhys;
3972 }
3973
3974 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3975 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3976 }
3977 else
3978 {
3979 /* Non-nested paging case, just use the hypervisor's CR3. */
3980 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3981
3982 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3983 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3984 }
3985 AssertRCReturn(rc, rc);
3986
3987 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3988 }
3989
3990 /*
3991 * Guest CR4.
3992 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3993 */
3994 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3995 {
3996 Assert(!(pMixedCtx->cr4 >> 32));
3997 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3998
3999 /* The guest's view of its CR4 is unblemished. */
4000 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4001 AssertRCReturn(rc, rc);
4002 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4003
4004 /* Setup VT-x's view of the guest CR4. */
4005 /*
4006 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4007 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4008 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4009 */
4010 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4011 {
4012 Assert(pVM->hm.s.vmx.pRealModeTSS);
4013 Assert(PDMVmmDevHeapIsEnabled(pVM));
4014 u32GuestCR4 &= ~X86_CR4_VME;
4015 }
4016
4017 if (pVM->hm.s.fNestedPaging)
4018 {
4019 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4020 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4021 {
4022 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4023 u32GuestCR4 |= X86_CR4_PSE;
4024 /* Our identity mapping is a 32-bit page directory. */
4025 u32GuestCR4 &= ~X86_CR4_PAE;
4026 }
4027 /* else use guest CR4.*/
4028 }
4029 else
4030 {
4031 /*
4032 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4033 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4034 */
4035 switch (pVCpu->hm.s.enmShadowMode)
4036 {
4037 case PGMMODE_REAL: /* Real-mode. */
4038 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4039 case PGMMODE_32_BIT: /* 32-bit paging. */
4040 {
4041 u32GuestCR4 &= ~X86_CR4_PAE;
4042 break;
4043 }
4044
4045 case PGMMODE_PAE: /* PAE paging. */
4046 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4047 {
4048 u32GuestCR4 |= X86_CR4_PAE;
4049 break;
4050 }
4051
4052 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4053 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4054#ifdef VBOX_ENABLE_64_BITS_GUESTS
4055 break;
4056#endif
4057 default:
4058 AssertFailed();
4059 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4060 }
4061 }
4062
4063 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4064 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4065 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4066 u32GuestCR4 |= uSetCR4;
4067 u32GuestCR4 &= uZapCR4;
4068
4069 /* Write VT-x's view of the guest CR4 into the VMCS. */
4070 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4071 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4072 AssertRCReturn(rc, rc);
4073
4074 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4075 uint32_t u32CR4Mask = X86_CR4_VME
4076 | X86_CR4_PAE
4077 | X86_CR4_PGE
4078 | X86_CR4_PSE
4079 | X86_CR4_VMXE;
4080 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4081 u32CR4Mask |= X86_CR4_OSXSAVE;
4082 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4083 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4084 AssertRCReturn(rc, rc);
4085
4086 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4087 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4088
4089 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4090 }
4091 return rc;
4092}
4093
4094
4095/**
4096 * Loads the guest debug registers into the guest-state area in the VMCS.
4097 *
4098 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4099 *
4100 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4101 *
4102 * @returns VBox status code.
4103 * @param pVCpu The cross context virtual CPU structure.
4104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4105 * out-of-sync. Make sure to update the required fields
4106 * before using them.
4107 *
4108 * @remarks No-long-jump zone!!!
4109 */
4110static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4111{
4112 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4113 return VINF_SUCCESS;
4114
4115#ifdef VBOX_STRICT
4116 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4118 {
4119 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4120 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4121 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4122 }
4123#endif
4124
4125 int rc;
4126 PVM pVM = pVCpu->CTX_SUFF(pVM);
4127 bool fSteppingDB = false;
4128 bool fInterceptMovDRx = false;
4129 if (pVCpu->hm.s.fSingleInstruction)
4130 {
4131 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4132 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4133 {
4134 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4135 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4136 AssertRCReturn(rc, rc);
4137 Assert(fSteppingDB == false);
4138 }
4139 else
4140 {
4141 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4142 pVCpu->hm.s.fClearTrapFlag = true;
4143 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4144 fSteppingDB = true;
4145 }
4146 }
4147
4148 if ( fSteppingDB
4149 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4150 {
4151 /*
4152 * Use the combined guest and host DRx values found in the hypervisor
4153 * register set because the debugger has breakpoints active or someone
4154 * is single stepping on the host side without a monitor trap flag.
4155 *
4156 * Note! DBGF expects a clean DR6 state before executing guest code.
4157 */
4158#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4159 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4160 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4161 {
4162 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4163 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4164 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4165 }
4166 else
4167#endif
4168 if (!CPUMIsHyperDebugStateActive(pVCpu))
4169 {
4170 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4171 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4172 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4173 }
4174
4175 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4176 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4177 AssertRCReturn(rc, rc);
4178
4179 pVCpu->hm.s.fUsingHyperDR7 = true;
4180 fInterceptMovDRx = true;
4181 }
4182 else
4183 {
4184 /*
4185 * If the guest has enabled debug registers, we need to load them prior to
4186 * executing guest code so they'll trigger at the right time.
4187 */
4188 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4189 {
4190#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4191 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4192 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4193 {
4194 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4195 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4196 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4197 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4198 }
4199 else
4200#endif
4201 if (!CPUMIsGuestDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4204 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4205 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4207 }
4208 Assert(!fInterceptMovDRx);
4209 }
4210 /*
4211 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4212 * must intercept #DB in order to maintain a correct DR6 guest value, and
4213 * because we need to intercept it to prevent nested #DBs from hanging the
4214 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4215 */
4216#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4217 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4218 && !CPUMIsGuestDebugStateActive(pVCpu))
4219#else
4220 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4221#endif
4222 {
4223 fInterceptMovDRx = true;
4224 }
4225
4226 /* Update guest DR7. */
4227 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4228 AssertRCReturn(rc, rc);
4229
4230 pVCpu->hm.s.fUsingHyperDR7 = false;
4231 }
4232
4233 /*
4234 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4235 */
4236 if (fInterceptMovDRx)
4237 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4238 else
4239 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4240 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4241 AssertRCReturn(rc, rc);
4242
4243 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4244 return VINF_SUCCESS;
4245}
4246
4247
4248#ifdef VBOX_STRICT
4249/**
4250 * Strict function to validate segment registers.
4251 *
4252 * @remarks ASSUMES CR0 is up to date.
4253 */
4254static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4255{
4256 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4257 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4258 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4259 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4260 && ( !CPUMIsGuestInRealModeEx(pCtx)
4261 && !CPUMIsGuestInV86ModeEx(pCtx)))
4262 {
4263 /* Protected mode checks */
4264 /* CS */
4265 Assert(pCtx->cs.Attr.n.u1Present);
4266 Assert(!(pCtx->cs.Attr.u & 0xf00));
4267 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4268 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4269 || !(pCtx->cs.Attr.n.u1Granularity));
4270 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4271 || (pCtx->cs.Attr.n.u1Granularity));
4272 /* CS cannot be loaded with NULL in protected mode. */
4273 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4274 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4275 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4276 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4277 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4278 else
4279 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4280 /* SS */
4281 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4282 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4283 if ( !(pCtx->cr0 & X86_CR0_PE)
4284 || pCtx->cs.Attr.n.u4Type == 3)
4285 {
4286 Assert(!pCtx->ss.Attr.n.u2Dpl);
4287 }
4288 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4289 {
4290 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4291 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4292 Assert(pCtx->ss.Attr.n.u1Present);
4293 Assert(!(pCtx->ss.Attr.u & 0xf00));
4294 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4295 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4296 || !(pCtx->ss.Attr.n.u1Granularity));
4297 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4298 || (pCtx->ss.Attr.n.u1Granularity));
4299 }
4300 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4301 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4302 {
4303 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4304 Assert(pCtx->ds.Attr.n.u1Present);
4305 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4306 Assert(!(pCtx->ds.Attr.u & 0xf00));
4307 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4308 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4309 || !(pCtx->ds.Attr.n.u1Granularity));
4310 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4311 || (pCtx->ds.Attr.n.u1Granularity));
4312 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4313 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4314 }
4315 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4316 {
4317 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4318 Assert(pCtx->es.Attr.n.u1Present);
4319 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4320 Assert(!(pCtx->es.Attr.u & 0xf00));
4321 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4322 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4323 || !(pCtx->es.Attr.n.u1Granularity));
4324 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4325 || (pCtx->es.Attr.n.u1Granularity));
4326 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4327 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4328 }
4329 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4330 {
4331 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4332 Assert(pCtx->fs.Attr.n.u1Present);
4333 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4334 Assert(!(pCtx->fs.Attr.u & 0xf00));
4335 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4336 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4337 || !(pCtx->fs.Attr.n.u1Granularity));
4338 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4339 || (pCtx->fs.Attr.n.u1Granularity));
4340 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4342 }
4343 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4344 {
4345 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4346 Assert(pCtx->gs.Attr.n.u1Present);
4347 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4348 Assert(!(pCtx->gs.Attr.u & 0xf00));
4349 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4350 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4351 || !(pCtx->gs.Attr.n.u1Granularity));
4352 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4353 || (pCtx->gs.Attr.n.u1Granularity));
4354 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4355 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4356 }
4357 /* 64-bit capable CPUs. */
4358# if HC_ARCH_BITS == 64
4359 Assert(!(pCtx->cs.u64Base >> 32));
4360 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4361 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4362 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4363# endif
4364 }
4365 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4366 || ( CPUMIsGuestInRealModeEx(pCtx)
4367 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4368 {
4369 /* Real and v86 mode checks. */
4370 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4371 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4372 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4373 {
4374 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4375 }
4376 else
4377 {
4378 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4379 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4380 }
4381
4382 /* CS */
4383 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4384 Assert(pCtx->cs.u32Limit == 0xffff);
4385 Assert(u32CSAttr == 0xf3);
4386 /* SS */
4387 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4388 Assert(pCtx->ss.u32Limit == 0xffff);
4389 Assert(u32SSAttr == 0xf3);
4390 /* DS */
4391 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4392 Assert(pCtx->ds.u32Limit == 0xffff);
4393 Assert(u32DSAttr == 0xf3);
4394 /* ES */
4395 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4396 Assert(pCtx->es.u32Limit == 0xffff);
4397 Assert(u32ESAttr == 0xf3);
4398 /* FS */
4399 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4400 Assert(pCtx->fs.u32Limit == 0xffff);
4401 Assert(u32FSAttr == 0xf3);
4402 /* GS */
4403 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4404 Assert(pCtx->gs.u32Limit == 0xffff);
4405 Assert(u32GSAttr == 0xf3);
4406 /* 64-bit capable CPUs. */
4407# if HC_ARCH_BITS == 64
4408 Assert(!(pCtx->cs.u64Base >> 32));
4409 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4410 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4411 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu The cross context virtual CPU structure.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4436 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4437 AssertRCReturn(rc, rc);
4438
4439 uint32_t u32Access = pSelReg->Attr.u;
4440 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4441 {
4442 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4443 u32Access = 0xf3;
4444 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4445 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4446 }
4447 else
4448 {
4449 /*
4450 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4451 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4452 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4453 * loaded in protected-mode have their attribute as 0.
4454 */
4455 if (!u32Access)
4456 u32Access = X86DESCATTR_UNUSABLE;
4457 }
4458
4459 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4460 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4461 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4462
4463 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4464 AssertRCReturn(rc, rc);
4465 return rc;
4466}
4467
4468
4469/**
4470 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4471 * into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4476 * out-of-sync. Make sure to update the required fields
4477 * before using them.
4478 *
4479 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4480 * @remarks No-long-jump zone!!!
4481 */
4482static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4483{
4484 int rc = VERR_INTERNAL_ERROR_5;
4485 PVM pVM = pVCpu->CTX_SUFF(pVM);
4486
4487 /*
4488 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4489 */
4490 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4491 {
4492 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 {
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4497 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4498 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4501 }
4502
4503#ifdef VBOX_WITH_REM
4504 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4505 {
4506 Assert(pVM->hm.s.vmx.pRealModeTSS);
4507 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4508 if ( pVCpu->hm.s.vmx.fWasInRealMode
4509 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4510 {
4511 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4512 in real-mode (e.g. OpenBSD 4.0) */
4513 REMFlushTBs(pVM);
4514 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4515 pVCpu->hm.s.vmx.fWasInRealMode = false;
4516 }
4517 }
4518#endif
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4520 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4523 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4526 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4529 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4532 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4535 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4536 AssertRCReturn(rc, rc);
4537
4538#ifdef VBOX_STRICT
4539 /* Validate. */
4540 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4541#endif
4542
4543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4544 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4545 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4552 {
4553 /*
4554 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4555 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4556 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4557 */
4558 uint16_t u16Sel = 0;
4559 uint32_t u32Limit = 0;
4560 uint64_t u64Base = 0;
4561 uint32_t u32AccessRights = 0;
4562
4563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4564 {
4565 u16Sel = pMixedCtx->tr.Sel;
4566 u32Limit = pMixedCtx->tr.u32Limit;
4567 u64Base = pMixedCtx->tr.u64Base;
4568 u32AccessRights = pMixedCtx->tr.Attr.u;
4569 }
4570 else
4571 {
4572 Assert(pVM->hm.s.vmx.pRealModeTSS);
4573 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4574
4575 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4576 RTGCPHYS GCPhys;
4577 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4578 AssertRCReturn(rc, rc);
4579
4580 X86DESCATTR DescAttr;
4581 DescAttr.u = 0;
4582 DescAttr.n.u1Present = 1;
4583 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4584
4585 u16Sel = 0;
4586 u32Limit = HM_VTX_TSS_SIZE;
4587 u64Base = GCPhys; /* in real-mode phys = virt. */
4588 u32AccessRights = DescAttr.u;
4589 }
4590
4591 /* Validate. */
4592 Assert(!(u16Sel & RT_BIT(2)));
4593 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4594 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4595 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4596 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4597 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4598 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4599 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4600 Assert( (u32Limit & 0xfff) == 0xfff
4601 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4602 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4603 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4604
4605 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4606 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4607 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 AssertRCReturn(rc, rc);
4610
4611 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4612 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4613 }
4614
4615 /*
4616 * Guest GDTR.
4617 */
4618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4619 {
4620 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4621 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4622 AssertRCReturn(rc, rc);
4623
4624 /* Validate. */
4625 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4626
4627 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4628 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4629 }
4630
4631 /*
4632 * Guest LDTR.
4633 */
4634 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4635 {
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pMixedCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pMixedCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4674 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4675 AssertRCReturn(rc, rc);
4676
4677 /* Validate. */
4678 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4679
4680 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4681 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4682 }
4683
4684 return VINF_SUCCESS;
4685}
4686
4687
4688/**
4689 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4690 * areas.
4691 *
4692 * These MSRs will automatically be loaded to the host CPU on every successful
4693 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4694 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4695 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4696 *
4697 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4698 *
4699 * @returns VBox status code.
4700 * @param pVCpu The cross context virtual CPU structure.
4701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4702 * out-of-sync. Make sure to update the required fields
4703 * before using them.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4717 {
4718 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4719#if HC_ARCH_BITS == 32
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4723 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4724 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4726 AssertRCReturn(rc, rc);
4727# ifdef LOG_ENABLED
4728 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4729 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4730 {
4731 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4732 pMsr->u64Value));
4733 }
4734# endif
4735 }
4736#endif
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4746 {
4747 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4758 {
4759 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4760 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4761 }
4762
4763 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4764 {
4765 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4766 {
4767 /*
4768 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4769 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4770 */
4771 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4772 {
4773 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4774 AssertRCReturn(rc,rc);
4775 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4776 }
4777 else
4778 {
4779 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4780 NULL /* pfAddedAndUpdated */);
4781 AssertRCReturn(rc, rc);
4782
4783 /* We need to intercept reads too, see @bugref{7386#c16}. */
4784 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4785 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4786 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4787 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4788 }
4789 }
4790 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4791 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4792 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4793 }
4794
4795 return VINF_SUCCESS;
4796}
4797
4798
4799/**
4800 * Loads the guest activity state into the guest-state area in the VMCS.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu The cross context virtual CPU structure.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 *
4808 * @remarks No-long-jump zone!!!
4809 */
4810static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811{
4812 NOREF(pMixedCtx);
4813 /** @todo See if we can make use of other states, e.g.
4814 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4815 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4816 {
4817 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4818 AssertRCReturn(rc, rc);
4819
4820 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4821 }
4822 return VINF_SUCCESS;
4823}
4824
4825
4826/**
4827 * Sets up the appropriate function to run guest code.
4828 *
4829 * @returns VBox status code.
4830 * @param pVCpu The cross context virtual CPU structure.
4831 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4832 * out-of-sync. Make sure to update the required fields
4833 * before using them.
4834 *
4835 * @remarks No-long-jump zone!!!
4836 */
4837static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4838{
4839 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4840 {
4841#ifndef VBOX_ENABLE_64_BITS_GUESTS
4842 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4843#endif
4844 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4845#if HC_ARCH_BITS == 32
4846 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4847 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4848 {
4849 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4850 {
4851 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4852 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4853 | HM_CHANGED_VMX_ENTRY_CTLS
4854 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4855 }
4856 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4857
4858 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4859 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4860 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4861 }
4862#else
4863 /* 64-bit host. */
4864 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4865#endif
4866 }
4867 else
4868 {
4869 /* Guest is not in long mode, use the 32-bit handler. */
4870#if HC_ARCH_BITS == 32
4871 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4872 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4873 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4874 {
4875 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4876 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4877 | HM_CHANGED_VMX_ENTRY_CTLS
4878 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4879 }
4880# ifdef VBOX_ENABLE_64_BITS_GUESTS
4881 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}.
4882 * Except if Real-on-V86 is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4883 * state where it's safe to use the 32-bit switcher again.
4884 */
4885 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4886 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4887
4888 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4889 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4890 else
4891 {
4892 Assert(!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active);
4893 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4894 }
4895# else
4896 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4897# endif
4898#else
4899 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4900#endif
4901 }
4902 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4903 return VINF_SUCCESS;
4904}
4905
4906
4907/**
4908 * Wrapper for running the guest code in VT-x.
4909 *
4910 * @returns VBox status code, no informational status codes.
4911 * @param pVM The cross context VM structure.
4912 * @param pVCpu The cross context virtual CPU structure.
4913 * @param pCtx Pointer to the guest-CPU context.
4914 *
4915 * @remarks No-long-jump zone!!!
4916 */
4917DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4918{
4919 /*
4920 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4921 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4922 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4923 */
4924 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4925 /** @todo Add stats for resume vs launch. */
4926#ifdef VBOX_WITH_KERNEL_USING_XMM
4927 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4928#else
4929 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4930#endif
4931 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4932 return rc;
4933}
4934
4935
4936/**
4937 * Reports world-switch error and dumps some useful debug info.
4938 *
4939 * @param pVM The cross context VM structure.
4940 * @param pVCpu The cross context virtual CPU structure.
4941 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4942 * @param pCtx Pointer to the guest-CPU context.
4943 * @param pVmxTransient Pointer to the VMX transient structure (only
4944 * exitReason updated).
4945 */
4946static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4947{
4948 Assert(pVM);
4949 Assert(pVCpu);
4950 Assert(pCtx);
4951 Assert(pVmxTransient);
4952 HMVMX_ASSERT_PREEMPT_SAFE();
4953
4954 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4955 switch (rcVMRun)
4956 {
4957 case VERR_VMX_INVALID_VMXON_PTR:
4958 AssertFailed();
4959 break;
4960 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4961 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4962 {
4963 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4964 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4965 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4966 AssertRC(rc);
4967
4968 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4969 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4970 Cannot do it here as we may have been long preempted. */
4971
4972#ifdef VBOX_STRICT
4973 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4974 pVmxTransient->uExitReason));
4975 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4976 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4977 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4978 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4979 else
4980 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4981 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4982 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4983
4984 /* VMX control bits. */
4985 uint32_t u32Val;
4986 uint64_t u64Val;
4987 RTHCUINTREG uHCReg;
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4996 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4997 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4998 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4999 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5000 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5001 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5002 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5003 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5004 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5005 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5006 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5007 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5008 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5009 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5010 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5011 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5014 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5015 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5016 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5017 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5018 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5019 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5021 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5022 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5023 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5024 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5025 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5026 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5027 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5028 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5029 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5030
5031 /* Guest bits. */
5032 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5033 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5034 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5035 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5036 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5037 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5040
5041 /* Host bits. */
5042 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5043 Log4(("Host CR0 %#RHr\n", uHCReg));
5044 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5045 Log4(("Host CR3 %#RHr\n", uHCReg));
5046 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5047 Log4(("Host CR4 %#RHr\n", uHCReg));
5048
5049 RTGDTR HostGdtr;
5050 PCX86DESCHC pDesc;
5051 ASMGetGDTR(&HostGdtr);
5052 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5053 Log4(("Host CS %#08x\n", u32Val));
5054 if (u32Val < HostGdtr.cbGdt)
5055 {
5056 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5057 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5058 }
5059
5060 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5061 Log4(("Host DS %#08x\n", u32Val));
5062 if (u32Val < HostGdtr.cbGdt)
5063 {
5064 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5065 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5066 }
5067
5068 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5069 Log4(("Host ES %#08x\n", u32Val));
5070 if (u32Val < HostGdtr.cbGdt)
5071 {
5072 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5073 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5074 }
5075
5076 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5077 Log4(("Host FS %#08x\n", u32Val));
5078 if (u32Val < HostGdtr.cbGdt)
5079 {
5080 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5081 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5082 }
5083
5084 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5085 Log4(("Host GS %#08x\n", u32Val));
5086 if (u32Val < HostGdtr.cbGdt)
5087 {
5088 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5089 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5090 }
5091
5092 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5093 Log4(("Host SS %#08x\n", u32Val));
5094 if (u32Val < HostGdtr.cbGdt)
5095 {
5096 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5097 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5098 }
5099
5100 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5101 Log4(("Host TR %#08x\n", u32Val));
5102 if (u32Val < HostGdtr.cbGdt)
5103 {
5104 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5105 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5106 }
5107
5108 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5109 Log4(("Host TR Base %#RHv\n", uHCReg));
5110 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5111 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5112 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5113 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5114 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5115 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5116 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5117 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5119 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5121 Log4(("Host RSP %#RHv\n", uHCReg));
5122 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5123 Log4(("Host RIP %#RHv\n", uHCReg));
5124# if HC_ARCH_BITS == 64
5125 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5126 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5127 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5128 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5129 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5130 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5131# endif
5132#endif /* VBOX_STRICT */
5133 break;
5134 }
5135
5136 default:
5137 /* Impossible */
5138 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5139 break;
5140 }
5141 NOREF(pVM); NOREF(pCtx);
5142}
5143
5144
5145#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5146#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5147# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5148#endif
5149#ifdef VBOX_STRICT
5150static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5151{
5152 switch (idxField)
5153 {
5154 case VMX_VMCS_GUEST_RIP:
5155 case VMX_VMCS_GUEST_RSP:
5156 case VMX_VMCS_GUEST_SYSENTER_EIP:
5157 case VMX_VMCS_GUEST_SYSENTER_ESP:
5158 case VMX_VMCS_GUEST_GDTR_BASE:
5159 case VMX_VMCS_GUEST_IDTR_BASE:
5160 case VMX_VMCS_GUEST_CS_BASE:
5161 case VMX_VMCS_GUEST_DS_BASE:
5162 case VMX_VMCS_GUEST_ES_BASE:
5163 case VMX_VMCS_GUEST_FS_BASE:
5164 case VMX_VMCS_GUEST_GS_BASE:
5165 case VMX_VMCS_GUEST_SS_BASE:
5166 case VMX_VMCS_GUEST_LDTR_BASE:
5167 case VMX_VMCS_GUEST_TR_BASE:
5168 case VMX_VMCS_GUEST_CR3:
5169 return true;
5170 }
5171 return false;
5172}
5173
5174static bool hmR0VmxIsValidReadField(uint32_t idxField)
5175{
5176 switch (idxField)
5177 {
5178 /* Read-only fields. */
5179 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5180 return true;
5181 }
5182 /* Remaining readable fields should also be writable. */
5183 return hmR0VmxIsValidWriteField(idxField);
5184}
5185#endif /* VBOX_STRICT */
5186
5187
5188/**
5189 * Executes the specified handler in 64-bit mode.
5190 *
5191 * @returns VBox status code (no informational status codes).
5192 * @param pVM The cross context VM structure.
5193 * @param pVCpu The cross context virtual CPU structure.
5194 * @param pCtx Pointer to the guest CPU context.
5195 * @param enmOp The operation to perform.
5196 * @param cParams Number of parameters.
5197 * @param paParam Array of 32-bit parameters.
5198 */
5199VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5200 uint32_t cParams, uint32_t *paParam)
5201{
5202 NOREF(pCtx);
5203
5204 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5205 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5206 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5207 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5208
5209#ifdef VBOX_STRICT
5210 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5211 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5212
5213 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5214 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5215#endif
5216
5217 /* Disable interrupts. */
5218 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5219
5220#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5221 RTCPUID idHostCpu = RTMpCpuId();
5222 CPUMR0SetLApic(pVCpu, idHostCpu);
5223#endif
5224
5225 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5226 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5227
5228 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5229 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5230
5231 /* Leave VMX Root Mode. */
5232 VMXDisable();
5233
5234 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5235
5236 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5237 CPUMSetHyperEIP(pVCpu, enmOp);
5238 for (int i = (int)cParams - 1; i >= 0; i--)
5239 CPUMPushHyper(pVCpu, paParam[i]);
5240
5241 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5242
5243 /* Call the switcher. */
5244 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5245 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5246
5247 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5248 /* Make sure the VMX instructions don't cause #UD faults. */
5249 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5250
5251 /* Re-enter VMX Root Mode */
5252 int rc2 = VMXEnable(HCPhysCpuPage);
5253 if (RT_FAILURE(rc2))
5254 {
5255 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5256 ASMSetFlags(fOldEFlags);
5257 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5258 return rc2;
5259 }
5260
5261 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5262 AssertRC(rc2);
5263 Assert(!(ASMGetFlags() & X86_EFL_IF));
5264 ASMSetFlags(fOldEFlags);
5265 return rc;
5266}
5267
5268
5269/**
5270 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5271 * supporting 64-bit guests.
5272 *
5273 * @returns VBox status code.
5274 * @param fResume Whether to VMLAUNCH or VMRESUME.
5275 * @param pCtx Pointer to the guest-CPU context.
5276 * @param pCache Pointer to the VMCS cache.
5277 * @param pVM The cross context VM structure.
5278 * @param pVCpu The cross context virtual CPU structure.
5279 */
5280DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5281{
5282 NOREF(fResume);
5283
5284 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5285 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5286
5287#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5288 pCache->uPos = 1;
5289 pCache->interPD = PGMGetInterPaeCR3(pVM);
5290 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5291#endif
5292
5293#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5294 pCache->TestIn.HCPhysCpuPage = 0;
5295 pCache->TestIn.HCPhysVmcs = 0;
5296 pCache->TestIn.pCache = 0;
5297 pCache->TestOut.HCPhysVmcs = 0;
5298 pCache->TestOut.pCache = 0;
5299 pCache->TestOut.pCtx = 0;
5300 pCache->TestOut.eflags = 0;
5301#else
5302 NOREF(pCache);
5303#endif
5304
5305 uint32_t aParam[10];
5306 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5307 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5308 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5309 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5310 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5311 aParam[5] = 0;
5312 aParam[6] = VM_RC_ADDR(pVM, pVM);
5313 aParam[7] = 0;
5314 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5315 aParam[9] = 0;
5316
5317#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5318 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5319 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5320#endif
5321 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5322
5323#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5324 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5325 Assert(pCtx->dr[4] == 10);
5326 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5327#endif
5328
5329#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5330 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5331 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5332 pVCpu->hm.s.vmx.HCPhysVmcs));
5333 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5334 pCache->TestOut.HCPhysVmcs));
5335 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5336 pCache->TestOut.pCache));
5337 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5338 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5339 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5340 pCache->TestOut.pCtx));
5341 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5342#endif
5343 return rc;
5344}
5345
5346
5347/**
5348 * Initialize the VMCS-Read cache.
5349 *
5350 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5351 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5352 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5353 * (those that have a 32-bit FULL & HIGH part).
5354 *
5355 * @returns VBox status code.
5356 * @param pVM The cross context VM structure.
5357 * @param pVCpu The cross context virtual CPU structure.
5358 */
5359static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5360{
5361#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5362{ \
5363 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5364 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5365 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5366 ++cReadFields; \
5367}
5368
5369 AssertPtr(pVM);
5370 AssertPtr(pVCpu);
5371 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5372 uint32_t cReadFields = 0;
5373
5374 /*
5375 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5376 * and serve to indicate exceptions to the rules.
5377 */
5378
5379 /* Guest-natural selector base fields. */
5380#if 0
5381 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5384#endif
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5397#if 0
5398 /* Unused natural width guest-state fields. */
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5401#endif
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5404
5405 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5406#if 0
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5409 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5410 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5416#endif
5417
5418 /* Natural width guest-state fields. */
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5420#if 0
5421 /* Currently unused field. */
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5423#endif
5424
5425 if (pVM->hm.s.fNestedPaging)
5426 {
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5428 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5429 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5430 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5431 }
5432 else
5433 {
5434 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5435 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5436 }
5437
5438#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5439 return VINF_SUCCESS;
5440}
5441
5442
5443/**
5444 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5445 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5446 * darwin, running 64-bit guests).
5447 *
5448 * @returns VBox status code.
5449 * @param pVCpu The cross context virtual CPU structure.
5450 * @param idxField The VMCS field encoding.
5451 * @param u64Val 16, 32 or 64-bit value.
5452 */
5453VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5454{
5455 int rc;
5456 switch (idxField)
5457 {
5458 /*
5459 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5460 */
5461 /* 64-bit Control fields. */
5462 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5463 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5464 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5465 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5466 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5467 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5468 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5469 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5470 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5471 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5472 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5473 case VMX_VMCS64_CTRL_EPTP_FULL:
5474 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5475 /* 64-bit Guest-state fields. */
5476 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5477 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5478 case VMX_VMCS64_GUEST_PAT_FULL:
5479 case VMX_VMCS64_GUEST_EFER_FULL:
5480 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5481 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5482 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5483 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5484 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5485 /* 64-bit Host-state fields. */
5486 case VMX_VMCS64_HOST_PAT_FULL:
5487 case VMX_VMCS64_HOST_EFER_FULL:
5488 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5489 {
5490 rc = VMXWriteVmcs32(idxField, u64Val);
5491 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5492 break;
5493 }
5494
5495 /*
5496 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5497 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5498 */
5499 /* Natural-width Guest-state fields. */
5500 case VMX_VMCS_GUEST_CR3:
5501 case VMX_VMCS_GUEST_ES_BASE:
5502 case VMX_VMCS_GUEST_CS_BASE:
5503 case VMX_VMCS_GUEST_SS_BASE:
5504 case VMX_VMCS_GUEST_DS_BASE:
5505 case VMX_VMCS_GUEST_FS_BASE:
5506 case VMX_VMCS_GUEST_GS_BASE:
5507 case VMX_VMCS_GUEST_LDTR_BASE:
5508 case VMX_VMCS_GUEST_TR_BASE:
5509 case VMX_VMCS_GUEST_GDTR_BASE:
5510 case VMX_VMCS_GUEST_IDTR_BASE:
5511 case VMX_VMCS_GUEST_RSP:
5512 case VMX_VMCS_GUEST_RIP:
5513 case VMX_VMCS_GUEST_SYSENTER_ESP:
5514 case VMX_VMCS_GUEST_SYSENTER_EIP:
5515 {
5516 if (!(u64Val >> 32))
5517 {
5518 /* If this field is 64-bit, VT-x will zero out the top bits. */
5519 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5520 }
5521 else
5522 {
5523 /* Assert that only the 32->64 switcher case should ever come here. */
5524 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5525 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5526 }
5527 break;
5528 }
5529
5530 default:
5531 {
5532 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5533 rc = VERR_INVALID_PARAMETER;
5534 break;
5535 }
5536 }
5537 AssertRCReturn(rc, rc);
5538 return rc;
5539}
5540
5541
5542/**
5543 * Queue up a VMWRITE by using the VMCS write cache.
5544 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5545 *
5546 * @param pVCpu The cross context virtual CPU structure.
5547 * @param idxField The VMCS field encoding.
5548 * @param u64Val 16, 32 or 64-bit value.
5549 */
5550VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5551{
5552 AssertPtr(pVCpu);
5553 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5554
5555 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5556 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5557
5558 /* Make sure there are no duplicates. */
5559 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5560 {
5561 if (pCache->Write.aField[i] == idxField)
5562 {
5563 pCache->Write.aFieldVal[i] = u64Val;
5564 return VINF_SUCCESS;
5565 }
5566 }
5567
5568 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5569 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5570 pCache->Write.cValidEntries++;
5571 return VINF_SUCCESS;
5572}
5573#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5574
5575
5576/**
5577 * Sets up the usage of TSC-offsetting and updates the VMCS.
5578 *
5579 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5580 * VMX preemption timer.
5581 *
5582 * @returns VBox status code.
5583 * @param pVM The cross context VM structure.
5584 * @param pVCpu The cross context virtual CPU structure.
5585 *
5586 * @remarks No-long-jump zone!!!
5587 */
5588static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5589{
5590 int rc;
5591 bool fOffsettedTsc;
5592 bool fParavirtTsc;
5593 if (pVM->hm.s.vmx.fUsePreemptTimer)
5594 {
5595 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5596 &fOffsettedTsc, &fParavirtTsc);
5597
5598 /* Make sure the returned values have sane upper and lower boundaries. */
5599 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5600 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5601 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5602 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5603
5604 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5605 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5606 }
5607 else
5608 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5609
5610 /** @todo later optimize this to be done elsewhere and not before every
5611 * VM-entry. */
5612 if (fParavirtTsc)
5613 {
5614 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5615 information before every VM-entry, hence disable it for performance sake. */
5616#if 0
5617 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5618 AssertRC(rc);
5619#endif
5620 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5621 }
5622
5623 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5624 {
5625 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5626 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5627
5628 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5629 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5630 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5631 }
5632 else
5633 {
5634 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5635 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5636 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5637 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5638 }
5639}
5640
5641
5642/**
5643 * Determines if an exception is a contributory exception.
5644 *
5645 * Contributory exceptions are ones which can cause double-faults unless the
5646 * original exception was a benign exception. Page-fault is intentionally not
5647 * included here as it's a conditional contributory exception.
5648 *
5649 * @returns true if the exception is contributory, false otherwise.
5650 * @param uVector The exception vector.
5651 */
5652DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5653{
5654 switch (uVector)
5655 {
5656 case X86_XCPT_GP:
5657 case X86_XCPT_SS:
5658 case X86_XCPT_NP:
5659 case X86_XCPT_TS:
5660 case X86_XCPT_DE:
5661 return true;
5662 default:
5663 break;
5664 }
5665 return false;
5666}
5667
5668
5669/**
5670 * Sets an event as a pending event to be injected into the guest.
5671 *
5672 * @param pVCpu The cross context virtual CPU structure.
5673 * @param u32IntInfo The VM-entry interruption-information field.
5674 * @param cbInstr The VM-entry instruction length in bytes (for software
5675 * interrupts, exceptions and privileged software
5676 * exceptions).
5677 * @param u32ErrCode The VM-entry exception error code.
5678 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5679 * page-fault.
5680 *
5681 * @remarks Statistics counter assumes this is a guest event being injected or
5682 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5683 * always incremented.
5684 */
5685DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5686 RTGCUINTPTR GCPtrFaultAddress)
5687{
5688 Assert(!pVCpu->hm.s.Event.fPending);
5689 pVCpu->hm.s.Event.fPending = true;
5690 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5691 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5692 pVCpu->hm.s.Event.cbInstr = cbInstr;
5693 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5694}
5695
5696
5697/**
5698 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5699 *
5700 * @param pVCpu The cross context virtual CPU structure.
5701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5702 * out-of-sync. Make sure to update the required fields
5703 * before using them.
5704 */
5705DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5706{
5707 NOREF(pMixedCtx);
5708 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5710 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5711 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5712}
5713
5714
5715/**
5716 * Handle a condition that occurred while delivering an event through the guest
5717 * IDT.
5718 *
5719 * @returns Strict VBox status code (i.e. informational status codes too).
5720 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5721 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5722 * to continue execution of the guest which will delivery the \#DF.
5723 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5724 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5725 *
5726 * @param pVCpu The cross context virtual CPU structure.
5727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5728 * out-of-sync. Make sure to update the required fields
5729 * before using them.
5730 * @param pVmxTransient Pointer to the VMX transient structure.
5731 *
5732 * @remarks No-long-jump zone!!!
5733 */
5734static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5735{
5736 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5737
5738 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5739 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5740
5741 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5742 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5743 {
5744 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5745 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5746
5747 typedef enum
5748 {
5749 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5750 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5751 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5752 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5753 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5754 } VMXREFLECTXCPT;
5755
5756 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5757 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5758 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5759 {
5760 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5761 {
5762 enmReflect = VMXREFLECTXCPT_XCPT;
5763#ifdef VBOX_STRICT
5764 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5765 && uExitVector == X86_XCPT_PF)
5766 {
5767 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5768 }
5769#endif
5770 if ( uExitVector == X86_XCPT_PF
5771 && uIdtVector == X86_XCPT_PF)
5772 {
5773 pVmxTransient->fVectoringDoublePF = true;
5774 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5775 }
5776 else if ( uExitVector == X86_XCPT_AC
5777 && uIdtVector == X86_XCPT_AC)
5778 {
5779 enmReflect = VMXREFLECTXCPT_HANG;
5780 Log4(("IDT: Nested #AC - Bad guest\n"));
5781 }
5782 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5783 && hmR0VmxIsContributoryXcpt(uExitVector)
5784 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5785 || uIdtVector == X86_XCPT_PF))
5786 {
5787 enmReflect = VMXREFLECTXCPT_DF;
5788 }
5789 else if (uIdtVector == X86_XCPT_DF)
5790 enmReflect = VMXREFLECTXCPT_TF;
5791 }
5792 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5793 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5794 {
5795 /*
5796 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5797 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5798 */
5799 enmReflect = VMXREFLECTXCPT_XCPT;
5800
5801 if (uExitVector == X86_XCPT_PF)
5802 {
5803 pVmxTransient->fVectoringPF = true;
5804 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5805 }
5806 }
5807 }
5808 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5809 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5810 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5811 {
5812 /*
5813 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5814 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5815 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5816 */
5817 enmReflect = VMXREFLECTXCPT_XCPT;
5818 }
5819
5820 /*
5821 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5822 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5823 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5824 *
5825 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5826 */
5827 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5828 && enmReflect == VMXREFLECTXCPT_XCPT
5829 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5830 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5831 {
5832 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5833 }
5834
5835 switch (enmReflect)
5836 {
5837 case VMXREFLECTXCPT_XCPT:
5838 {
5839 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5840 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5841 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5842
5843 uint32_t u32ErrCode = 0;
5844 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5845 {
5846 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5847 AssertRCReturn(rc2, rc2);
5848 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5849 }
5850
5851 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5852 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5853 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5854 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5855 rcStrict = VINF_SUCCESS;
5856 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5857 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5858
5859 break;
5860 }
5861
5862 case VMXREFLECTXCPT_DF:
5863 {
5864 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5865 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5866 rcStrict = VINF_HM_DOUBLE_FAULT;
5867 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5868 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5869
5870 break;
5871 }
5872
5873 case VMXREFLECTXCPT_TF:
5874 {
5875 rcStrict = VINF_EM_RESET;
5876 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5877 uExitVector));
5878 break;
5879 }
5880
5881 case VMXREFLECTXCPT_HANG:
5882 {
5883 rcStrict = VERR_EM_GUEST_CPU_HANG;
5884 break;
5885 }
5886
5887 default:
5888 Assert(rcStrict == VINF_SUCCESS);
5889 break;
5890 }
5891 }
5892 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5893 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5894 && uExitVector != X86_XCPT_DF
5895 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5896 {
5897 /*
5898 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5899 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5900 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5901 */
5902 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5903 {
5904 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5905 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5906 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5907 }
5908 }
5909
5910 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5911 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5912 return rcStrict;
5913}
5914
5915
5916/**
5917 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5918 *
5919 * @returns VBox status code.
5920 * @param pVCpu The cross context virtual CPU structure.
5921 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5922 * out-of-sync. Make sure to update the required fields
5923 * before using them.
5924 *
5925 * @remarks No-long-jump zone!!!
5926 */
5927static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5928{
5929 NOREF(pMixedCtx);
5930
5931 /*
5932 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5933 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5934 */
5935 VMMRZCallRing3Disable(pVCpu);
5936 HM_DISABLE_PREEMPT();
5937
5938 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5939 {
5940 uint32_t uVal = 0;
5941 uint32_t uShadow = 0;
5942 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5943 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5944 AssertRCReturn(rc, rc);
5945
5946 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5947 CPUMSetGuestCR0(pVCpu, uVal);
5948 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5949 }
5950
5951 HM_RESTORE_PREEMPT();
5952 VMMRZCallRing3Enable(pVCpu);
5953 return VINF_SUCCESS;
5954}
5955
5956
5957/**
5958 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5959 *
5960 * @returns VBox status code.
5961 * @param pVCpu The cross context virtual CPU structure.
5962 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5963 * out-of-sync. Make sure to update the required fields
5964 * before using them.
5965 *
5966 * @remarks No-long-jump zone!!!
5967 */
5968static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5969{
5970 NOREF(pMixedCtx);
5971
5972 int rc = VINF_SUCCESS;
5973 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5974 {
5975 uint32_t uVal = 0;
5976 uint32_t uShadow = 0;
5977 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5978 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5979 AssertRCReturn(rc, rc);
5980
5981 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5982 CPUMSetGuestCR4(pVCpu, uVal);
5983 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5984 }
5985 return rc;
5986}
5987
5988
5989/**
5990 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5991 *
5992 * @returns VBox status code.
5993 * @param pVCpu The cross context virtual CPU structure.
5994 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5995 * out-of-sync. Make sure to update the required fields
5996 * before using them.
5997 *
5998 * @remarks No-long-jump zone!!!
5999 */
6000static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6001{
6002 int rc = VINF_SUCCESS;
6003 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6004 {
6005 uint64_t u64Val = 0;
6006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6007 AssertRCReturn(rc, rc);
6008
6009 pMixedCtx->rip = u64Val;
6010 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6011 }
6012 return rc;
6013}
6014
6015
6016/**
6017 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6018 *
6019 * @returns VBox status code.
6020 * @param pVCpu The cross context virtual CPU structure.
6021 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6022 * out-of-sync. Make sure to update the required fields
6023 * before using them.
6024 *
6025 * @remarks No-long-jump zone!!!
6026 */
6027static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6028{
6029 int rc = VINF_SUCCESS;
6030 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6031 {
6032 uint64_t u64Val = 0;
6033 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6034 AssertRCReturn(rc, rc);
6035
6036 pMixedCtx->rsp = u64Val;
6037 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6038 }
6039 return rc;
6040}
6041
6042
6043/**
6044 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6045 *
6046 * @returns VBox status code.
6047 * @param pVCpu The cross context virtual CPU structure.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6057 {
6058 uint32_t uVal = 0;
6059 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6060 AssertRCReturn(rc, rc);
6061
6062 pMixedCtx->eflags.u32 = uVal;
6063 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6064 {
6065 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6066 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6067
6068 pMixedCtx->eflags.Bits.u1VM = 0;
6069 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6070 }
6071
6072 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6073 }
6074 return VINF_SUCCESS;
6075}
6076
6077
6078/**
6079 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6080 * guest-CPU context.
6081 */
6082DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6083{
6084 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6085 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6086 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6087 return rc;
6088}
6089
6090
6091/**
6092 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6093 * from the guest-state area in the VMCS.
6094 *
6095 * @param pVCpu The cross context virtual CPU structure.
6096 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6097 * out-of-sync. Make sure to update the required fields
6098 * before using them.
6099 *
6100 * @remarks No-long-jump zone!!!
6101 */
6102static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6103{
6104 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6105 {
6106 uint32_t uIntrState = 0;
6107 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6108 AssertRC(rc);
6109
6110 if (!uIntrState)
6111 {
6112 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6113 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6114
6115 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6116 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6117 }
6118 else
6119 {
6120 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6121 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6122 {
6123 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6124 AssertRC(rc);
6125 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6126 AssertRC(rc);
6127
6128 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6129 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6130 }
6131 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6133
6134 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6135 {
6136 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6137 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6138 }
6139 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6140 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6141 }
6142
6143 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6144 }
6145}
6146
6147
6148/**
6149 * Saves the guest's activity state.
6150 *
6151 * @returns VBox status code.
6152 * @param pVCpu The cross context virtual CPU structure.
6153 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6154 * out-of-sync. Make sure to update the required fields
6155 * before using them.
6156 *
6157 * @remarks No-long-jump zone!!!
6158 */
6159static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6160{
6161 NOREF(pMixedCtx);
6162 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6163 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6164 return VINF_SUCCESS;
6165}
6166
6167
6168/**
6169 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6170 * the current VMCS into the guest-CPU context.
6171 *
6172 * @returns VBox status code.
6173 * @param pVCpu The cross context virtual CPU structure.
6174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6175 * out-of-sync. Make sure to update the required fields
6176 * before using them.
6177 *
6178 * @remarks No-long-jump zone!!!
6179 */
6180static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6181{
6182 int rc = VINF_SUCCESS;
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6184 {
6185 uint32_t u32Val = 0;
6186 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6187 pMixedCtx->SysEnter.cs = u32Val;
6188 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6189 }
6190
6191 uint64_t u64Val = 0;
6192 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6193 {
6194 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6195 pMixedCtx->SysEnter.eip = u64Val;
6196 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6197 }
6198 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6199 {
6200 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6201 pMixedCtx->SysEnter.esp = u64Val;
6202 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6203 }
6204 return rc;
6205}
6206
6207
6208/**
6209 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6210 * the CPU back into the guest-CPU context.
6211 *
6212 * @returns VBox status code.
6213 * @param pVCpu The cross context virtual CPU structure.
6214 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6215 * out-of-sync. Make sure to update the required fields
6216 * before using them.
6217 *
6218 * @remarks No-long-jump zone!!!
6219 */
6220static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6221{
6222 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6223 VMMRZCallRing3Disable(pVCpu);
6224 HM_DISABLE_PREEMPT();
6225
6226 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6227 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6228 {
6229 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6230 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6231 }
6232
6233 HM_RESTORE_PREEMPT();
6234 VMMRZCallRing3Enable(pVCpu);
6235
6236 return VINF_SUCCESS;
6237}
6238
6239
6240/**
6241 * Saves the auto load/store'd guest MSRs from the current VMCS into
6242 * the guest-CPU context.
6243 *
6244 * @returns VBox status code.
6245 * @param pVCpu The cross context virtual CPU structure.
6246 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6247 * out-of-sync. Make sure to update the required fields
6248 * before using them.
6249 *
6250 * @remarks No-long-jump zone!!!
6251 */
6252static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6253{
6254 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6255 return VINF_SUCCESS;
6256
6257 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6258 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6259 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6260 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6261 {
6262 switch (pMsr->u32Msr)
6263 {
6264 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6265 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6266 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6267 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6268 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6269 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6270 break;
6271
6272 default:
6273 {
6274 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6275 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6276 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6277 }
6278 }
6279 }
6280
6281 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6282 return VINF_SUCCESS;
6283}
6284
6285
6286/**
6287 * Saves the guest control registers from the current VMCS into the guest-CPU
6288 * context.
6289 *
6290 * @returns VBox status code.
6291 * @param pVCpu The cross context virtual CPU structure.
6292 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6293 * out-of-sync. Make sure to update the required fields
6294 * before using them.
6295 *
6296 * @remarks No-long-jump zone!!!
6297 */
6298static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6299{
6300 /* Guest CR0. Guest FPU. */
6301 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6302 AssertRCReturn(rc, rc);
6303
6304 /* Guest CR4. */
6305 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6306 AssertRCReturn(rc, rc);
6307
6308 /* Guest CR2 - updated always during the world-switch or in #PF. */
6309 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6310 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6311 {
6312 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6313 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6314
6315 PVM pVM = pVCpu->CTX_SUFF(pVM);
6316 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6317 || ( pVM->hm.s.fNestedPaging
6318 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6319 {
6320 uint64_t u64Val = 0;
6321 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6322 if (pMixedCtx->cr3 != u64Val)
6323 {
6324 CPUMSetGuestCR3(pVCpu, u64Val);
6325 if (VMMRZCallRing3IsEnabled(pVCpu))
6326 {
6327 PGMUpdateCR3(pVCpu, u64Val);
6328 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6329 }
6330 else
6331 {
6332 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6333 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6334 }
6335 }
6336
6337 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6338 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6339 {
6340 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6341 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6342 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6343 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6344 AssertRCReturn(rc, rc);
6345
6346 if (VMMRZCallRing3IsEnabled(pVCpu))
6347 {
6348 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6349 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6350 }
6351 else
6352 {
6353 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6354 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6355 }
6356 }
6357 }
6358
6359 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6360 }
6361
6362 /*
6363 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6364 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6365 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6366 *
6367 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6368 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6369 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6370 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6371 *
6372 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6373 */
6374 if (VMMRZCallRing3IsEnabled(pVCpu))
6375 {
6376 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6377 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6378
6379 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6380 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6381
6382 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6383 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6384 }
6385
6386 return rc;
6387}
6388
6389
6390/**
6391 * Reads a guest segment register from the current VMCS into the guest-CPU
6392 * context.
6393 *
6394 * @returns VBox status code.
6395 * @param pVCpu The cross context virtual CPU structure.
6396 * @param idxSel Index of the selector in the VMCS.
6397 * @param idxLimit Index of the segment limit in the VMCS.
6398 * @param idxBase Index of the segment base in the VMCS.
6399 * @param idxAccess Index of the access rights of the segment in the VMCS.
6400 * @param pSelReg Pointer to the segment selector.
6401 *
6402 * @remarks No-long-jump zone!!!
6403 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6404 * macro as that takes care of whether to read from the VMCS cache or
6405 * not.
6406 */
6407DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6408 PCPUMSELREG pSelReg)
6409{
6410 NOREF(pVCpu);
6411
6412 uint32_t u32Val = 0;
6413 int rc = VMXReadVmcs32(idxSel, &u32Val);
6414 AssertRCReturn(rc, rc);
6415 pSelReg->Sel = (uint16_t)u32Val;
6416 pSelReg->ValidSel = (uint16_t)u32Val;
6417 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6418
6419 rc = VMXReadVmcs32(idxLimit, &u32Val);
6420 AssertRCReturn(rc, rc);
6421 pSelReg->u32Limit = u32Val;
6422
6423 uint64_t u64Val = 0;
6424 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6425 AssertRCReturn(rc, rc);
6426 pSelReg->u64Base = u64Val;
6427
6428 rc = VMXReadVmcs32(idxAccess, &u32Val);
6429 AssertRCReturn(rc, rc);
6430 pSelReg->Attr.u = u32Val;
6431
6432 /*
6433 * If VT-x marks the segment as unusable, most other bits remain undefined:
6434 * - For CS the L, D and G bits have meaning.
6435 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6436 * - For the remaining data segments no bits are defined.
6437 *
6438 * The present bit and the unusable bit has been observed to be set at the
6439 * same time (the selector was supposed to be invalid as we started executing
6440 * a V8086 interrupt in ring-0).
6441 *
6442 * What should be important for the rest of the VBox code, is that the P bit is
6443 * cleared. Some of the other VBox code recognizes the unusable bit, but
6444 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6445 * safe side here, we'll strip off P and other bits we don't care about. If
6446 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6447 *
6448 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6449 */
6450 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6451 {
6452 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6453
6454 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6455 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6456 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6457
6458 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6459#ifdef DEBUG_bird
6460 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6461 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6462 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6463#endif
6464 }
6465 return VINF_SUCCESS;
6466}
6467
6468
6469#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6470# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6471 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6472 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6473#else
6474# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6475 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6476 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6477#endif
6478
6479
6480/**
6481 * Saves the guest segment registers from the current VMCS into the guest-CPU
6482 * context.
6483 *
6484 * @returns VBox status code.
6485 * @param pVCpu The cross context virtual CPU structure.
6486 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6487 * out-of-sync. Make sure to update the required fields
6488 * before using them.
6489 *
6490 * @remarks No-long-jump zone!!!
6491 */
6492static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6493{
6494 /* Guest segment registers. */
6495 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6496 {
6497 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6498 AssertRCReturn(rc, rc);
6499
6500 rc = VMXLOCAL_READ_SEG(CS, cs);
6501 rc |= VMXLOCAL_READ_SEG(SS, ss);
6502 rc |= VMXLOCAL_READ_SEG(DS, ds);
6503 rc |= VMXLOCAL_READ_SEG(ES, es);
6504 rc |= VMXLOCAL_READ_SEG(FS, fs);
6505 rc |= VMXLOCAL_READ_SEG(GS, gs);
6506 AssertRCReturn(rc, rc);
6507
6508 /* Restore segment attributes for real-on-v86 mode hack. */
6509 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6510 {
6511 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6512 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6513 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6514 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6515 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6516 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6517 }
6518 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6519 }
6520
6521 return VINF_SUCCESS;
6522}
6523
6524
6525/**
6526 * Saves the guest descriptor table registers and task register from the current
6527 * VMCS into the guest-CPU context.
6528 *
6529 * @returns VBox status code.
6530 * @param pVCpu The cross context virtual CPU structure.
6531 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6532 * out-of-sync. Make sure to update the required fields
6533 * before using them.
6534 *
6535 * @remarks No-long-jump zone!!!
6536 */
6537static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6538{
6539 int rc = VINF_SUCCESS;
6540
6541 /* Guest LDTR. */
6542 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6543 {
6544 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6545 AssertRCReturn(rc, rc);
6546 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6547 }
6548
6549 /* Guest GDTR. */
6550 uint64_t u64Val = 0;
6551 uint32_t u32Val = 0;
6552 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6553 {
6554 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6555 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6556 pMixedCtx->gdtr.pGdt = u64Val;
6557 pMixedCtx->gdtr.cbGdt = u32Val;
6558 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6559 }
6560
6561 /* Guest IDTR. */
6562 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6563 {
6564 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6565 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6566 pMixedCtx->idtr.pIdt = u64Val;
6567 pMixedCtx->idtr.cbIdt = u32Val;
6568 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6569 }
6570
6571 /* Guest TR. */
6572 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6573 {
6574 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6575 AssertRCReturn(rc, rc);
6576
6577 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6578 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6579 {
6580 rc = VMXLOCAL_READ_SEG(TR, tr);
6581 AssertRCReturn(rc, rc);
6582 }
6583 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6584 }
6585 return rc;
6586}
6587
6588#undef VMXLOCAL_READ_SEG
6589
6590
6591/**
6592 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6593 * context.
6594 *
6595 * @returns VBox status code.
6596 * @param pVCpu The cross context virtual CPU structure.
6597 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6598 * out-of-sync. Make sure to update the required fields
6599 * before using them.
6600 *
6601 * @remarks No-long-jump zone!!!
6602 */
6603static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6604{
6605 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6606 {
6607 if (!pVCpu->hm.s.fUsingHyperDR7)
6608 {
6609 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6610 uint32_t u32Val;
6611 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6612 pMixedCtx->dr[7] = u32Val;
6613 }
6614
6615 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6616 }
6617 return VINF_SUCCESS;
6618}
6619
6620
6621/**
6622 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6623 *
6624 * @returns VBox status code.
6625 * @param pVCpu The cross context virtual CPU structure.
6626 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6627 * out-of-sync. Make sure to update the required fields
6628 * before using them.
6629 *
6630 * @remarks No-long-jump zone!!!
6631 */
6632static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6633{
6634 NOREF(pMixedCtx);
6635
6636 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6637 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6638 return VINF_SUCCESS;
6639}
6640
6641
6642/**
6643 * Saves the entire guest state from the currently active VMCS into the
6644 * guest-CPU context.
6645 *
6646 * This essentially VMREADs all guest-data.
6647 *
6648 * @returns VBox status code.
6649 * @param pVCpu The cross context virtual CPU structure.
6650 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6651 * out-of-sync. Make sure to update the required fields
6652 * before using them.
6653 */
6654static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6655{
6656 Assert(pVCpu);
6657 Assert(pMixedCtx);
6658
6659 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6660 return VINF_SUCCESS;
6661
6662 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6663 again on the ring-3 callback path, there is no real need to. */
6664 if (VMMRZCallRing3IsEnabled(pVCpu))
6665 VMMR0LogFlushDisable(pVCpu);
6666 else
6667 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6668 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6669
6670 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6671 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6672
6673 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6674 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6675
6676 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6677 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6678
6679 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6686 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6687
6688 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6690
6691 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6693
6694 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6696
6697 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6699
6700 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6701 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6702 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6703
6704 if (VMMRZCallRing3IsEnabled(pVCpu))
6705 VMMR0LogFlushEnable(pVCpu);
6706
6707 return VINF_SUCCESS;
6708}
6709
6710
6711/**
6712 * Saves basic guest registers needed for IEM instruction execution.
6713 *
6714 * @returns VBox status code (OR-able).
6715 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6716 * @param pMixedCtx Pointer to the CPU context of the guest.
6717 * @param fMemory Whether the instruction being executed operates on
6718 * memory or not. Only CR0 is synced up if clear.
6719 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6720 */
6721static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6722{
6723 /*
6724 * We assume all general purpose registers other than RSP are available.
6725 *
6726 * RIP is a must, as it will be incremented or otherwise changed.
6727 *
6728 * RFLAGS are always required to figure the CPL.
6729 *
6730 * RSP isn't always required, however it's a GPR, so frequently required.
6731 *
6732 * SS and CS are the only segment register needed if IEM doesn't do memory
6733 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6734 *
6735 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6736 * be required for memory accesses.
6737 *
6738 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6739 */
6740 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6741 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6742 if (fNeedRsp)
6743 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6744 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6745 if (!fMemory)
6746 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6747 else
6748 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6749 AssertRCReturn(rc, rc);
6750 return rc;
6751}
6752
6753
6754/**
6755 * Ensures that we've got a complete basic guest-context.
6756 *
6757 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6758 * is for the interpreter.
6759 *
6760 * @returns VBox status code.
6761 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6762 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6763 * needing to be synced in.
6764 * @thread EMT(pVCpu)
6765 */
6766VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6767{
6768 /* Note! Since this is only applicable to VT-x, the implementation is placed
6769 in the VT-x part of the sources instead of the generic stuff. */
6770 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6771 {
6772 /* For now, imply that the caller might change everything too. */
6773 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6774 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6775 }
6776 return VINF_SUCCESS;
6777}
6778
6779
6780/**
6781 * Check per-VM and per-VCPU force flag actions that require us to go back to
6782 * ring-3 for one reason or another.
6783 *
6784 * @returns Strict VBox status code (i.e. informational status codes too)
6785 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6786 * ring-3.
6787 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6788 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6789 * interrupts)
6790 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6791 * all EMTs to be in ring-3.
6792 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6793 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6794 * to the EM loop.
6795 *
6796 * @param pVM The cross context VM structure.
6797 * @param pVCpu The cross context virtual CPU structure.
6798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6799 * out-of-sync. Make sure to update the required fields
6800 * before using them.
6801 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6802 */
6803static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6804{
6805 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6806
6807 /*
6808 * Anything pending? Should be more likely than not if we're doing a good job.
6809 */
6810 if ( !fStepping
6811 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6812 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6813 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6814 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6815 return VINF_SUCCESS;
6816
6817 /* We need the control registers now, make sure the guest-CPU context is updated. */
6818 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6819 AssertRCReturn(rc3, rc3);
6820
6821 /* Pending HM CR3 sync. */
6822 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6823 {
6824 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6825 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6826 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6827 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6828 }
6829
6830 /* Pending HM PAE PDPEs. */
6831 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6832 {
6833 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6834 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6835 }
6836
6837 /* Pending PGM C3 sync. */
6838 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6839 {
6840 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6841 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6842 if (rcStrict2 != VINF_SUCCESS)
6843 {
6844 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6845 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6846 return rcStrict2;
6847 }
6848 }
6849
6850 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6851 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6852 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6853 {
6854 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6855 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6856 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6857 return rc2;
6858 }
6859
6860 /* Pending VM request packets, such as hardware interrupts. */
6861 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6862 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6863 {
6864 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6865 return VINF_EM_PENDING_REQUEST;
6866 }
6867
6868 /* Pending PGM pool flushes. */
6869 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6870 {
6871 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6872 return VINF_PGM_POOL_FLUSH_PENDING;
6873 }
6874
6875 /* Pending DMA requests. */
6876 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6877 {
6878 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6879 return VINF_EM_RAW_TO_R3;
6880 }
6881
6882 return VINF_SUCCESS;
6883}
6884
6885
6886/**
6887 * Converts any TRPM trap into a pending HM event. This is typically used when
6888 * entering from ring-3 (not longjmp returns).
6889 *
6890 * @param pVCpu The cross context virtual CPU structure.
6891 */
6892static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6893{
6894 Assert(TRPMHasTrap(pVCpu));
6895 Assert(!pVCpu->hm.s.Event.fPending);
6896
6897 uint8_t uVector;
6898 TRPMEVENT enmTrpmEvent;
6899 RTGCUINT uErrCode;
6900 RTGCUINTPTR GCPtrFaultAddress;
6901 uint8_t cbInstr;
6902
6903 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6904 AssertRC(rc);
6905
6906 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6907 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6908 if (enmTrpmEvent == TRPM_TRAP)
6909 {
6910 switch (uVector)
6911 {
6912 case X86_XCPT_NMI:
6913 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6914 break;
6915
6916 case X86_XCPT_BP:
6917 case X86_XCPT_OF:
6918 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6919 break;
6920
6921 case X86_XCPT_PF:
6922 case X86_XCPT_DF:
6923 case X86_XCPT_TS:
6924 case X86_XCPT_NP:
6925 case X86_XCPT_SS:
6926 case X86_XCPT_GP:
6927 case X86_XCPT_AC:
6928 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6929 /* no break! */
6930 default:
6931 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6932 break;
6933 }
6934 }
6935 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6936 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6937 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6938 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6939 else
6940 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6941
6942 rc = TRPMResetTrap(pVCpu);
6943 AssertRC(rc);
6944 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6945 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6946
6947 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6948}
6949
6950
6951/**
6952 * Converts the pending HM event into a TRPM trap.
6953 *
6954 * @param pVCpu The cross context virtual CPU structure.
6955 */
6956static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6957{
6958 Assert(pVCpu->hm.s.Event.fPending);
6959
6960 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6961 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6962 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6963 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6964
6965 /* If a trap was already pending, we did something wrong! */
6966 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6967
6968 TRPMEVENT enmTrapType;
6969 switch (uVectorType)
6970 {
6971 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6972 enmTrapType = TRPM_HARDWARE_INT;
6973 break;
6974
6975 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6976 enmTrapType = TRPM_SOFTWARE_INT;
6977 break;
6978
6979 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6980 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6981 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6982 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6983 enmTrapType = TRPM_TRAP;
6984 break;
6985
6986 default:
6987 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6988 enmTrapType = TRPM_32BIT_HACK;
6989 break;
6990 }
6991
6992 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6993
6994 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6995 AssertRC(rc);
6996
6997 if (fErrorCodeValid)
6998 TRPMSetErrorCode(pVCpu, uErrorCode);
6999
7000 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7001 && uVector == X86_XCPT_PF)
7002 {
7003 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7004 }
7005 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7006 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7007 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7008 {
7009 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7010 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7011 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7012 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7013 }
7014
7015 /* Clear any pending events from the VMCS. */
7016 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7017 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7018
7019 /* We're now done converting the pending event. */
7020 pVCpu->hm.s.Event.fPending = false;
7021}
7022
7023
7024/**
7025 * Does the necessary state syncing before returning to ring-3 for any reason
7026 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7027 *
7028 * @returns VBox status code.
7029 * @param pVM The cross context VM structure.
7030 * @param pVCpu The cross context virtual CPU structure.
7031 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7032 * be out-of-sync. Make sure to update the required
7033 * fields before using them.
7034 * @param fSaveGuestState Whether to save the guest state or not.
7035 *
7036 * @remarks No-long-jmp zone!!!
7037 */
7038static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7039{
7040 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7041 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7042
7043 RTCPUID idCpu = RTMpCpuId();
7044 Log4Func(("HostCpuId=%u\n", idCpu));
7045
7046 /*
7047 * !!! IMPORTANT !!!
7048 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7049 */
7050
7051 /* Save the guest state if necessary. */
7052 if ( fSaveGuestState
7053 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7054 {
7055 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7056 AssertRCReturn(rc, rc);
7057 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7058 }
7059
7060 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7061 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7062 {
7063 if (fSaveGuestState)
7064 {
7065 /* We shouldn't reload CR0 without saving it first. */
7066 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7067 AssertRCReturn(rc, rc);
7068 }
7069 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7070 }
7071
7072 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7073#ifdef VBOX_STRICT
7074 if (CPUMIsHyperDebugStateActive(pVCpu))
7075 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7076#endif
7077 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7078 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7079 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7080 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7081
7082#if HC_ARCH_BITS == 64
7083 /* Restore host-state bits that VT-x only restores partially. */
7084 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7085 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7086 {
7087 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7088 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7089 }
7090 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7091#endif
7092
7093 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7094 if (pVCpu->hm.s.vmx.fLazyMsrs)
7095 {
7096 /* We shouldn't reload the guest MSRs without saving it first. */
7097 if (!fSaveGuestState)
7098 {
7099 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7100 AssertRCReturn(rc, rc);
7101 }
7102 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7103 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7104 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7105 }
7106
7107 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7108 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7109
7110 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7111 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7112 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7113 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7114 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7115 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7116 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7117 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7118
7119 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7120
7121 /** @todo This partially defeats the purpose of having preemption hooks.
7122 * The problem is, deregistering the hooks should be moved to a place that
7123 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7124 * context.
7125 */
7126 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7127 {
7128 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7129 AssertRCReturn(rc, rc);
7130
7131 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7132 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7133 }
7134 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7135 NOREF(idCpu);
7136
7137 return VINF_SUCCESS;
7138}
7139
7140
7141/**
7142 * Leaves the VT-x session.
7143 *
7144 * @returns VBox status code.
7145 * @param pVM The cross context VM structure.
7146 * @param pVCpu The cross context virtual CPU structure.
7147 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7148 * out-of-sync. Make sure to update the required fields
7149 * before using them.
7150 *
7151 * @remarks No-long-jmp zone!!!
7152 */
7153DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7154{
7155 HM_DISABLE_PREEMPT();
7156 HMVMX_ASSERT_CPU_SAFE();
7157 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7158 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7159
7160 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7161 and done this from the VMXR0ThreadCtxCallback(). */
7162 if (!pVCpu->hm.s.fLeaveDone)
7163 {
7164 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7165 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7166 pVCpu->hm.s.fLeaveDone = true;
7167 }
7168 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7169
7170 /*
7171 * !!! IMPORTANT !!!
7172 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7173 */
7174
7175 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7176 /** @todo Deregistering here means we need to VMCLEAR always
7177 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7178 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7179 VMMR0ThreadCtxHookDisable(pVCpu);
7180
7181 /* Leave HM context. This takes care of local init (term). */
7182 int rc = HMR0LeaveCpu(pVCpu);
7183
7184 HM_RESTORE_PREEMPT();
7185 return rc;
7186}
7187
7188
7189/**
7190 * Does the necessary state syncing before doing a longjmp to ring-3.
7191 *
7192 * @returns VBox status code.
7193 * @param pVM The cross context VM structure.
7194 * @param pVCpu The cross context virtual CPU structure.
7195 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7196 * out-of-sync. Make sure to update the required fields
7197 * before using them.
7198 *
7199 * @remarks No-long-jmp zone!!!
7200 */
7201DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7202{
7203 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7204}
7205
7206
7207/**
7208 * Take necessary actions before going back to ring-3.
7209 *
7210 * An action requires us to go back to ring-3. This function does the necessary
7211 * steps before we can safely return to ring-3. This is not the same as longjmps
7212 * to ring-3, this is voluntary and prepares the guest so it may continue
7213 * executing outside HM (recompiler/IEM).
7214 *
7215 * @returns VBox status code.
7216 * @param pVM The cross context VM structure.
7217 * @param pVCpu The cross context virtual CPU structure.
7218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7219 * out-of-sync. Make sure to update the required fields
7220 * before using them.
7221 * @param rcExit The reason for exiting to ring-3. Can be
7222 * VINF_VMM_UNKNOWN_RING3_CALL.
7223 */
7224static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7225{
7226 Assert(pVM);
7227 Assert(pVCpu);
7228 Assert(pMixedCtx);
7229 HMVMX_ASSERT_PREEMPT_SAFE();
7230
7231 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7232 {
7233 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7234 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7235 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7236 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7237 }
7238
7239 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7240 VMMRZCallRing3Disable(pVCpu);
7241 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7242
7243 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7244 if (pVCpu->hm.s.Event.fPending)
7245 {
7246 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7247 Assert(!pVCpu->hm.s.Event.fPending);
7248 }
7249
7250 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7251 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7252
7253 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7254 and if we're injecting an event we should have a TRPM trap pending. */
7255 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7256#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7257 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7258#endif
7259
7260 /* Save guest state and restore host state bits. */
7261 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7262 AssertRCReturn(rc, rc);
7263 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7264 /* Thread-context hooks are unregistered at this point!!! */
7265
7266 /* Sync recompiler state. */
7267 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7268 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7269 | CPUM_CHANGED_LDTR
7270 | CPUM_CHANGED_GDTR
7271 | CPUM_CHANGED_IDTR
7272 | CPUM_CHANGED_TR
7273 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7274 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7275 if ( pVM->hm.s.fNestedPaging
7276 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7277 {
7278 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7279 }
7280
7281 Assert(!pVCpu->hm.s.fClearTrapFlag);
7282
7283 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7284 if (rcExit != VINF_EM_RAW_INTERRUPT)
7285 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7286
7287 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7288
7289 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7290 VMMRZCallRing3RemoveNotification(pVCpu);
7291 VMMRZCallRing3Enable(pVCpu);
7292
7293 return rc;
7294}
7295
7296
7297/**
7298 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7299 * longjump to ring-3 and possibly get preempted.
7300 *
7301 * @returns VBox status code.
7302 * @param pVCpu The cross context virtual CPU structure.
7303 * @param enmOperation The operation causing the ring-3 longjump.
7304 * @param pvUser Opaque pointer to the guest-CPU context. The data
7305 * may be out-of-sync. Make sure to update the required
7306 * fields before using them.
7307 */
7308static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7309{
7310 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7311 {
7312 /*
7313 * !!! IMPORTANT !!!
7314 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7315 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7316 */
7317 VMMRZCallRing3RemoveNotification(pVCpu);
7318 VMMRZCallRing3Disable(pVCpu);
7319 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7320 RTThreadPreemptDisable(&PreemptState);
7321
7322 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7323 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7324
7325#if HC_ARCH_BITS == 64
7326 /* Restore host-state bits that VT-x only restores partially. */
7327 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7328 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7329 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7330 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7331#endif
7332 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7333 if (pVCpu->hm.s.vmx.fLazyMsrs)
7334 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7335
7336 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7337 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7338 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7339 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7340 {
7341 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7342 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7343 }
7344
7345 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7346 VMMR0ThreadCtxHookDisable(pVCpu);
7347 HMR0LeaveCpu(pVCpu);
7348 RTThreadPreemptRestore(&PreemptState);
7349 return VINF_SUCCESS;
7350 }
7351
7352 Assert(pVCpu);
7353 Assert(pvUser);
7354 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7355 HMVMX_ASSERT_PREEMPT_SAFE();
7356
7357 VMMRZCallRing3Disable(pVCpu);
7358 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7359
7360 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7361 enmOperation));
7362
7363 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7364 AssertRCReturn(rc, rc);
7365
7366 VMMRZCallRing3Enable(pVCpu);
7367 return VINF_SUCCESS;
7368}
7369
7370
7371/**
7372 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7373 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7374 *
7375 * @param pVCpu The cross context virtual CPU structure.
7376 */
7377DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7378{
7379 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7380 {
7381 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7382 {
7383 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7384 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7385 AssertRC(rc);
7386 Log4(("Setup interrupt-window exiting\n"));
7387 }
7388 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7389}
7390
7391
7392/**
7393 * Clears the interrupt-window exiting control in the VMCS.
7394 *
7395 * @param pVCpu The cross context virtual CPU structure.
7396 */
7397DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7398{
7399 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7400 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7401 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7402 AssertRC(rc);
7403 Log4(("Cleared interrupt-window exiting\n"));
7404}
7405
7406
7407/**
7408 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7409 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7410 *
7411 * @param pVCpu The cross context virtual CPU structure.
7412 */
7413DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7414{
7415 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7416 {
7417 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7418 {
7419 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7420 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7421 AssertRC(rc);
7422 Log4(("Setup NMI-window exiting\n"));
7423 }
7424 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7425}
7426
7427
7428/**
7429 * Clears the NMI-window exiting control in the VMCS.
7430 *
7431 * @param pVCpu The cross context virtual CPU structure.
7432 */
7433DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7434{
7435 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7436 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7437 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7438 AssertRC(rc);
7439 Log4(("Cleared NMI-window exiting\n"));
7440}
7441
7442
7443/**
7444 * Evaluates the event to be delivered to the guest and sets it as the pending
7445 * event.
7446 *
7447 * @returns The VT-x guest-interruptibility state.
7448 * @param pVCpu The cross context virtual CPU structure.
7449 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7450 * out-of-sync. Make sure to update the required fields
7451 * before using them.
7452 */
7453static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7454{
7455 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7456 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7457 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7458 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7459 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7460
7461 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7462 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7463 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7464 Assert(!TRPMHasTrap(pVCpu));
7465
7466#ifdef VBOX_WITH_NEW_APIC
7467 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7468 APICUpdatePendingInterrupts(pVCpu);
7469#endif
7470
7471 /*
7472 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7473 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7474 */
7475 /** @todo SMI. SMIs take priority over NMIs. */
7476 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7477 {
7478 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7479 if ( !pVCpu->hm.s.Event.fPending
7480 && !fBlockNmi
7481 && !fBlockSti
7482 && !fBlockMovSS)
7483 {
7484 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7485 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7486 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7487
7488 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7489 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7490 }
7491 else
7492 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7493 }
7494 /*
7495 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7496 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7497 */
7498 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7499 && !pVCpu->hm.s.fSingleInstruction)
7500 {
7501 Assert(!DBGFIsStepping(pVCpu));
7502 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7503 AssertRC(rc);
7504 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7505 if ( !pVCpu->hm.s.Event.fPending
7506 && !fBlockInt
7507 && !fBlockSti
7508 && !fBlockMovSS)
7509 {
7510 uint8_t u8Interrupt;
7511 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7512 if (RT_SUCCESS(rc))
7513 {
7514 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7515 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7516 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7517
7518 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7519 }
7520 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7521 {
7522 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7523 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7524 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7525 }
7526 else
7527 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7528 }
7529 else
7530 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7531 }
7532
7533 return uIntrState;
7534}
7535
7536
7537/**
7538 * Sets a pending-debug exception to be delivered to the guest if the guest is
7539 * single-stepping in the VMCS.
7540 *
7541 * @param pVCpu The cross context virtual CPU structure.
7542 */
7543DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7544{
7545 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7546 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7547 AssertRC(rc);
7548}
7549
7550
7551/**
7552 * Injects any pending events into the guest if the guest is in a state to
7553 * receive them.
7554 *
7555 * @returns Strict VBox status code (i.e. informational status codes too).
7556 * @param pVCpu The cross context virtual CPU structure.
7557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7558 * out-of-sync. Make sure to update the required fields
7559 * before using them.
7560 * @param uIntrState The VT-x guest-interruptibility state.
7561 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7562 * return VINF_EM_DBG_STEPPED if the event was
7563 * dispatched directly.
7564 */
7565static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7566{
7567 HMVMX_ASSERT_PREEMPT_SAFE();
7568 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7569
7570 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7571 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7572
7573 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7574 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7575 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7576 Assert(!TRPMHasTrap(pVCpu));
7577
7578 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7579 if (pVCpu->hm.s.Event.fPending)
7580 {
7581 /*
7582 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7583 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7584 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7585 *
7586 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7587 */
7588 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7589#ifdef VBOX_STRICT
7590 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7591 {
7592 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7593 Assert(!fBlockInt);
7594 Assert(!fBlockSti);
7595 Assert(!fBlockMovSS);
7596 }
7597 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7598 {
7599 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7600 Assert(!fBlockSti);
7601 Assert(!fBlockMovSS);
7602 Assert(!fBlockNmi);
7603 }
7604#endif
7605 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7606 (uint8_t)uIntType));
7607 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7608 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7609 fStepping, &uIntrState);
7610 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7611
7612 /* Update the interruptibility-state as it could have been changed by
7613 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7614 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7615 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7616
7617 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7618 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7619 else
7620 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7621 }
7622
7623 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7624 if ( fBlockSti
7625 || fBlockMovSS)
7626 {
7627 if (!pVCpu->hm.s.fSingleInstruction)
7628 {
7629 /*
7630 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7631 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7632 * See Intel spec. 27.3.4 "Saving Non-Register State".
7633 */
7634 Assert(!DBGFIsStepping(pVCpu));
7635 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7636 AssertRCReturn(rc2, rc2);
7637 if (pMixedCtx->eflags.Bits.u1TF)
7638 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7639 }
7640 else if (pMixedCtx->eflags.Bits.u1TF)
7641 {
7642 /*
7643 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7644 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7645 */
7646 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7647 uIntrState = 0;
7648 }
7649 }
7650
7651 /*
7652 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7653 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7654 */
7655 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7656 AssertRC(rc2);
7657
7658 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7659 NOREF(fBlockMovSS); NOREF(fBlockSti);
7660 return rcStrict;
7661}
7662
7663
7664/**
7665 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7666 *
7667 * @param pVCpu The cross context virtual CPU structure.
7668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7669 * out-of-sync. Make sure to update the required fields
7670 * before using them.
7671 */
7672DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7673{
7674 NOREF(pMixedCtx);
7675 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7676 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7677}
7678
7679
7680/**
7681 * Injects a double-fault (\#DF) exception into the VM.
7682 *
7683 * @returns Strict VBox status code (i.e. informational status codes too).
7684 * @param pVCpu The cross context virtual CPU structure.
7685 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7686 * out-of-sync. Make sure to update the required fields
7687 * before using them.
7688 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7689 * and should return VINF_EM_DBG_STEPPED if the event
7690 * is injected directly (register modified by us, not
7691 * by hardware on VM-entry).
7692 * @param puIntrState Pointer to the current guest interruptibility-state.
7693 * This interruptibility-state will be updated if
7694 * necessary. This cannot not be NULL.
7695 */
7696DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7697{
7698 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7699 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7700 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7701 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7702 fStepping, puIntrState);
7703}
7704
7705
7706/**
7707 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7708 *
7709 * @param pVCpu The cross context virtual CPU structure.
7710 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7711 * out-of-sync. Make sure to update the required fields
7712 * before using them.
7713 */
7714DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7715{
7716 NOREF(pMixedCtx);
7717 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7718 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7719 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7720}
7721
7722
7723/**
7724 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7725 *
7726 * @param pVCpu The cross context virtual CPU structure.
7727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7728 * out-of-sync. Make sure to update the required fields
7729 * before using them.
7730 * @param cbInstr The value of RIP that is to be pushed on the guest
7731 * stack.
7732 */
7733DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7734{
7735 NOREF(pMixedCtx);
7736 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7737 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7738 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7739}
7740
7741
7742/**
7743 * Injects a general-protection (\#GP) fault into the VM.
7744 *
7745 * @returns Strict VBox status code (i.e. informational status codes too).
7746 * @param pVCpu The cross context virtual CPU structure.
7747 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7748 * out-of-sync. Make sure to update the required fields
7749 * before using them.
7750 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7751 * mode, i.e. in real-mode it's not valid).
7752 * @param u32ErrorCode The error code associated with the \#GP.
7753 * @param fStepping Whether we're running in
7754 * hmR0VmxRunGuestCodeStep() and should return
7755 * VINF_EM_DBG_STEPPED if the event is injected
7756 * directly (register modified by us, not by
7757 * hardware on VM-entry).
7758 * @param puIntrState Pointer to the current guest interruptibility-state.
7759 * This interruptibility-state will be updated if
7760 * necessary. This cannot not be NULL.
7761 */
7762DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7763 bool fStepping, uint32_t *puIntrState)
7764{
7765 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7766 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7767 if (fErrorCodeValid)
7768 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7769 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7770 fStepping, puIntrState);
7771}
7772
7773
7774/**
7775 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7776 * VM.
7777 *
7778 * @param pVCpu The cross context virtual CPU structure.
7779 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7780 * out-of-sync. Make sure to update the required fields
7781 * before using them.
7782 * @param u32ErrorCode The error code associated with the \#GP.
7783 */
7784DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7785{
7786 NOREF(pMixedCtx);
7787 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7788 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7789 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7790 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7791}
7792
7793
7794/**
7795 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7796 *
7797 * @param pVCpu The cross context virtual CPU structure.
7798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7799 * out-of-sync. Make sure to update the required fields
7800 * before using them.
7801 * @param uVector The software interrupt vector number.
7802 * @param cbInstr The value of RIP that is to be pushed on the guest
7803 * stack.
7804 */
7805DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7806{
7807 NOREF(pMixedCtx);
7808 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7809 if ( uVector == X86_XCPT_BP
7810 || uVector == X86_XCPT_OF)
7811 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7812 else
7813 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7814 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7815}
7816
7817
7818/**
7819 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7820 * stack.
7821 *
7822 * @returns Strict VBox status code (i.e. informational status codes too).
7823 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7824 * @param pVM The cross context VM structure.
7825 * @param pMixedCtx Pointer to the guest-CPU context.
7826 * @param uValue The value to push to the guest stack.
7827 */
7828DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7829{
7830 /*
7831 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7832 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7833 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7834 */
7835 if (pMixedCtx->sp == 1)
7836 return VINF_EM_RESET;
7837 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7838 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7839 AssertRC(rc);
7840 return rc;
7841}
7842
7843
7844/**
7845 * Injects an event into the guest upon VM-entry by updating the relevant fields
7846 * in the VM-entry area in the VMCS.
7847 *
7848 * @returns Strict VBox status code (i.e. informational status codes too).
7849 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7850 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7851 *
7852 * @param pVCpu The cross context virtual CPU structure.
7853 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7854 * be out-of-sync. Make sure to update the required
7855 * fields before using them.
7856 * @param u64IntInfo The VM-entry interruption-information field.
7857 * @param cbInstr The VM-entry instruction length in bytes (for
7858 * software interrupts, exceptions and privileged
7859 * software exceptions).
7860 * @param u32ErrCode The VM-entry exception error code.
7861 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7862 * @param puIntrState Pointer to the current guest interruptibility-state.
7863 * This interruptibility-state will be updated if
7864 * necessary. This cannot not be NULL.
7865 * @param fStepping Whether we're running in
7866 * hmR0VmxRunGuestCodeStep() and should return
7867 * VINF_EM_DBG_STEPPED if the event is injected
7868 * directly (register modified by us, not by
7869 * hardware on VM-entry).
7870 *
7871 * @remarks Requires CR0!
7872 * @remarks No-long-jump zone!!!
7873 */
7874static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7875 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7876 uint32_t *puIntrState)
7877{
7878 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7879 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7880 Assert(puIntrState);
7881 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7882
7883 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7884 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7885
7886#ifdef VBOX_STRICT
7887 /* Validate the error-code-valid bit for hardware exceptions. */
7888 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7889 {
7890 switch (uVector)
7891 {
7892 case X86_XCPT_PF:
7893 case X86_XCPT_DF:
7894 case X86_XCPT_TS:
7895 case X86_XCPT_NP:
7896 case X86_XCPT_SS:
7897 case X86_XCPT_GP:
7898 case X86_XCPT_AC:
7899 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7900 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7901 /* fallthru */
7902 default:
7903 break;
7904 }
7905 }
7906#endif
7907
7908 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7909 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7910 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7911
7912 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7913
7914 /* We require CR0 to check if the guest is in real-mode. */
7915 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7916 AssertRCReturn(rc, rc);
7917
7918 /*
7919 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7920 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7921 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7922 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7923 */
7924 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7925 {
7926 PVM pVM = pVCpu->CTX_SUFF(pVM);
7927 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7928 {
7929 Assert(PDMVmmDevHeapIsEnabled(pVM));
7930 Assert(pVM->hm.s.vmx.pRealModeTSS);
7931
7932 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7933 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7934 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7935 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7936 AssertRCReturn(rc, rc);
7937 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7938
7939 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7940 size_t const cbIdtEntry = sizeof(X86IDTR16);
7941 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7942 {
7943 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7944 if (uVector == X86_XCPT_DF)
7945 return VINF_EM_RESET;
7946
7947 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7948 if (uVector == X86_XCPT_GP)
7949 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7950
7951 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7952 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7953 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7954 fStepping, puIntrState);
7955 }
7956
7957 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7958 uint16_t uGuestIp = pMixedCtx->ip;
7959 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7960 {
7961 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7962 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7963 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7964 }
7965 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7966 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7967
7968 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7969 X86IDTR16 IdtEntry;
7970 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7971 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7972 AssertRCReturn(rc, rc);
7973
7974 /* Construct the stack frame for the interrupt/exception handler. */
7975 VBOXSTRICTRC rcStrict;
7976 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7977 if (rcStrict == VINF_SUCCESS)
7978 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7979 if (rcStrict == VINF_SUCCESS)
7980 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7981
7982 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7983 if (rcStrict == VINF_SUCCESS)
7984 {
7985 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7986 pMixedCtx->rip = IdtEntry.offSel;
7987 pMixedCtx->cs.Sel = IdtEntry.uSel;
7988 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7989 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7990 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7991 && uVector == X86_XCPT_PF)
7992 pMixedCtx->cr2 = GCPtrFaultAddress;
7993
7994 /* If any other guest-state bits are changed here, make sure to update
7995 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7996 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7997 | HM_CHANGED_GUEST_RIP
7998 | HM_CHANGED_GUEST_RFLAGS
7999 | HM_CHANGED_GUEST_RSP);
8000
8001 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8002 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8003 {
8004 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8005 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8006 Log4(("Clearing inhibition due to STI.\n"));
8007 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8008 }
8009 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8010 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8011
8012 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8013 it, if we are returning to ring-3 before executing guest code. */
8014 pVCpu->hm.s.Event.fPending = false;
8015
8016 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8017 if (fStepping)
8018 rcStrict = VINF_EM_DBG_STEPPED;
8019 }
8020 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8021 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8022 return rcStrict;
8023 }
8024
8025 /*
8026 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8027 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8028 */
8029 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8030 }
8031
8032 /* Validate. */
8033 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8034 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8035 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8036
8037 /* Inject. */
8038 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8039 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8040 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8041 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8042
8043 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8044 && uVector == X86_XCPT_PF)
8045 pMixedCtx->cr2 = GCPtrFaultAddress;
8046
8047 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8048 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8049
8050 AssertRCReturn(rc, rc);
8051 return VINF_SUCCESS;
8052}
8053
8054
8055/**
8056 * Clears the interrupt-window exiting control in the VMCS and if necessary
8057 * clears the current event in the VMCS as well.
8058 *
8059 * @returns VBox status code.
8060 * @param pVCpu The cross context virtual CPU structure.
8061 *
8062 * @remarks Use this function only to clear events that have not yet been
8063 * delivered to the guest but are injected in the VMCS!
8064 * @remarks No-long-jump zone!!!
8065 */
8066static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8067{
8068 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8069
8070 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8071 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8072
8073 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8074 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8075}
8076
8077
8078/**
8079 * Enters the VT-x session.
8080 *
8081 * @returns VBox status code.
8082 * @param pVM The cross context VM structure.
8083 * @param pVCpu The cross context virtual CPU structure.
8084 * @param pCpu Pointer to the CPU info struct.
8085 */
8086VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8087{
8088 AssertPtr(pVM);
8089 AssertPtr(pVCpu);
8090 Assert(pVM->hm.s.vmx.fSupported);
8091 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8092 NOREF(pCpu); NOREF(pVM);
8093
8094 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8095 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8096
8097#ifdef VBOX_STRICT
8098 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8099 RTCCUINTREG uHostCR4 = ASMGetCR4();
8100 if (!(uHostCR4 & X86_CR4_VMXE))
8101 {
8102 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8103 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8104 }
8105#endif
8106
8107 /*
8108 * Load the VCPU's VMCS as the current (and active) one.
8109 */
8110 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8111 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8112 if (RT_FAILURE(rc))
8113 return rc;
8114
8115 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8116 pVCpu->hm.s.fLeaveDone = false;
8117 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8118
8119 return VINF_SUCCESS;
8120}
8121
8122
8123/**
8124 * The thread-context callback (only on platforms which support it).
8125 *
8126 * @param enmEvent The thread-context event.
8127 * @param pVCpu The cross context virtual CPU structure.
8128 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8129 * @thread EMT(pVCpu)
8130 */
8131VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8132{
8133 NOREF(fGlobalInit);
8134
8135 switch (enmEvent)
8136 {
8137 case RTTHREADCTXEVENT_OUT:
8138 {
8139 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8140 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8141 VMCPU_ASSERT_EMT(pVCpu);
8142
8143 PVM pVM = pVCpu->CTX_SUFF(pVM);
8144 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8145
8146 /* No longjmps (logger flushes, locks) in this fragile context. */
8147 VMMRZCallRing3Disable(pVCpu);
8148 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8149
8150 /*
8151 * Restore host-state (FPU, debug etc.)
8152 */
8153 if (!pVCpu->hm.s.fLeaveDone)
8154 {
8155 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8156 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8157 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8158 pVCpu->hm.s.fLeaveDone = true;
8159 }
8160
8161 /* Leave HM context, takes care of local init (term). */
8162 int rc = HMR0LeaveCpu(pVCpu);
8163 AssertRC(rc); NOREF(rc);
8164
8165 /* Restore longjmp state. */
8166 VMMRZCallRing3Enable(pVCpu);
8167 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8168 break;
8169 }
8170
8171 case RTTHREADCTXEVENT_IN:
8172 {
8173 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8174 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8175 VMCPU_ASSERT_EMT(pVCpu);
8176
8177 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8178 VMMRZCallRing3Disable(pVCpu);
8179 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8180
8181 /* Initialize the bare minimum state required for HM. This takes care of
8182 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8183 int rc = HMR0EnterCpu(pVCpu);
8184 AssertRC(rc);
8185 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8186
8187 /* Load the active VMCS as the current one. */
8188 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8189 {
8190 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8191 AssertRC(rc); NOREF(rc);
8192 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8193 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8194 }
8195 pVCpu->hm.s.fLeaveDone = false;
8196
8197 /* Restore longjmp state. */
8198 VMMRZCallRing3Enable(pVCpu);
8199 break;
8200 }
8201
8202 default:
8203 break;
8204 }
8205}
8206
8207
8208/**
8209 * Saves the host state in the VMCS host-state.
8210 * Sets up the VM-exit MSR-load area.
8211 *
8212 * The CPU state will be loaded from these fields on every successful VM-exit.
8213 *
8214 * @returns VBox status code.
8215 * @param pVM The cross context VM structure.
8216 * @param pVCpu The cross context virtual CPU structure.
8217 *
8218 * @remarks No-long-jump zone!!!
8219 */
8220static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8221{
8222 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8223
8224 int rc = VINF_SUCCESS;
8225 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8226 {
8227 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8228 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8229
8230 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8231 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8232
8233 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8234 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8235
8236 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8237 }
8238 return rc;
8239}
8240
8241
8242/**
8243 * Saves the host state in the VMCS host-state.
8244 *
8245 * @returns VBox status code.
8246 * @param pVM The cross context VM structure.
8247 * @param pVCpu The cross context virtual CPU structure.
8248 *
8249 * @remarks No-long-jump zone!!!
8250 */
8251VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8252{
8253 AssertPtr(pVM);
8254 AssertPtr(pVCpu);
8255
8256 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8257
8258 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8259 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8260 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8261 return hmR0VmxSaveHostState(pVM, pVCpu);
8262}
8263
8264
8265/**
8266 * Loads the guest state into the VMCS guest-state area.
8267 *
8268 * The will typically be done before VM-entry when the guest-CPU state and the
8269 * VMCS state may potentially be out of sync.
8270 *
8271 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8272 * VM-entry controls.
8273 * Sets up the appropriate VMX non-root function to execute guest code based on
8274 * the guest CPU mode.
8275 *
8276 * @returns VBox strict status code.
8277 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8278 * without unrestricted guest access and the VMMDev is not presently
8279 * mapped (e.g. EFI32).
8280 *
8281 * @param pVM The cross context VM structure.
8282 * @param pVCpu The cross context virtual CPU structure.
8283 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8284 * out-of-sync. Make sure to update the required fields
8285 * before using them.
8286 *
8287 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8288 * caller disables then again on successfull return. Confusing.)
8289 */
8290static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8291{
8292 AssertPtr(pVM);
8293 AssertPtr(pVCpu);
8294 AssertPtr(pMixedCtx);
8295 HMVMX_ASSERT_PREEMPT_SAFE();
8296
8297 VMMRZCallRing3Disable(pVCpu);
8298 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8299
8300 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8301
8302 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8303
8304 /* Determine real-on-v86 mode. */
8305 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8306 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8307 && CPUMIsGuestInRealModeEx(pMixedCtx))
8308 {
8309 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8310 }
8311
8312 /*
8313 * Load the guest-state into the VMCS.
8314 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8315 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8316 */
8317 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8318 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8319
8320 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8321 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8322 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8323
8324 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8325 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8326 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8327
8328 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8329 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8330
8331 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8332 if (rcStrict == VINF_SUCCESS)
8333 { /* likely */ }
8334 else
8335 {
8336 VMMRZCallRing3Enable(pVCpu);
8337 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8338 return rcStrict;
8339 }
8340
8341 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8342 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8343 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8344
8345 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8346 determine we don't have to swap EFER after all. */
8347 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8348 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8349
8350 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8351 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8352
8353 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8354 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8355
8356 /*
8357 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8358 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8359 */
8360 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8361 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8362
8363 /* Clear any unused and reserved bits. */
8364 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8365
8366 VMMRZCallRing3Enable(pVCpu);
8367
8368 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8369 return rc;
8370}
8371
8372
8373/**
8374 * Loads the state shared between the host and guest into the VMCS.
8375 *
8376 * @param pVM The cross context VM structure.
8377 * @param pVCpu The cross context virtual CPU structure.
8378 * @param pCtx Pointer to the guest-CPU context.
8379 *
8380 * @remarks No-long-jump zone!!!
8381 */
8382static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8383{
8384 NOREF(pVM);
8385
8386 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8387 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8388
8389 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8390 {
8391 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8392 AssertRC(rc);
8393 }
8394
8395 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8396 {
8397 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8398 AssertRC(rc);
8399
8400 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8401 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8402 {
8403 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8404 AssertRC(rc);
8405 }
8406 }
8407
8408 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8409 {
8410 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8411 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8412 }
8413
8414 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8415 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8416 {
8417 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8418 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8419 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8420 AssertRC(rc);
8421 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8422 }
8423
8424 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8425 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8426}
8427
8428
8429/**
8430 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8431 *
8432 * @returns Strict VBox status code (i.e. informational status codes too).
8433 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8434 * without unrestricted guest access and the VMMDev is not presently
8435 * mapped (e.g. EFI32).
8436 *
8437 * @param pVM The cross context VM structure.
8438 * @param pVCpu The cross context virtual CPU structure.
8439 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8440 * out-of-sync. Make sure to update the required fields
8441 * before using them.
8442 */
8443static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8444{
8445 HMVMX_ASSERT_PREEMPT_SAFE();
8446
8447 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8448#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8449 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8450#endif
8451
8452 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8453 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8454 {
8455 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8456 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8457 { /* likely */}
8458 else
8459 {
8460 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8461 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8462 }
8463 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8464 }
8465 else if (HMCPU_CF_VALUE(pVCpu))
8466 {
8467 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8468 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8469 { /* likely */}
8470 else
8471 {
8472 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8473 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8474 return rcStrict;
8475 }
8476 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8477 }
8478
8479 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8480 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8481 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8482 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8483 return rcStrict;
8484}
8485
8486
8487/**
8488 * Does the preparations before executing guest code in VT-x.
8489 *
8490 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8491 * recompiler/IEM. We must be cautious what we do here regarding committing
8492 * guest-state information into the VMCS assuming we assuredly execute the
8493 * guest in VT-x mode.
8494 *
8495 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8496 * the common-state (TRPM/forceflags), we must undo those changes so that the
8497 * recompiler/IEM can (and should) use them when it resumes guest execution.
8498 * Otherwise such operations must be done when we can no longer exit to ring-3.
8499 *
8500 * @returns Strict VBox status code (i.e. informational status codes too).
8501 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8502 * have been disabled.
8503 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8504 * double-fault into the guest.
8505 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8506 * dispatched directly.
8507 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8508 *
8509 * @param pVM The cross context VM structure.
8510 * @param pVCpu The cross context virtual CPU structure.
8511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8512 * out-of-sync. Make sure to update the required fields
8513 * before using them.
8514 * @param pVmxTransient Pointer to the VMX transient structure.
8515 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8516 * us ignore some of the reasons for returning to
8517 * ring-3, and return VINF_EM_DBG_STEPPED if event
8518 * dispatching took place.
8519 */
8520static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8521{
8522 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8523
8524#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8525 PGMRZDynMapFlushAutoSet(pVCpu);
8526#endif
8527
8528 /* Check force flag actions that might require us to go back to ring-3. */
8529 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8530 if (rcStrict == VINF_SUCCESS)
8531 { /* FFs doesn't get set all the time. */ }
8532 else
8533 return rcStrict;
8534
8535#ifndef IEM_VERIFICATION_MODE_FULL
8536 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8537 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8538 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8539 {
8540 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8541 RTGCPHYS GCPhysApicBase;
8542 GCPhysApicBase = pMixedCtx->msrApicBase;
8543 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8544
8545 /* Unalias any existing mapping. */
8546 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8547 AssertRCReturn(rc, rc);
8548
8549 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8550 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8551 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8552 AssertRCReturn(rc, rc);
8553
8554 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8555 }
8556#endif /* !IEM_VERIFICATION_MODE_FULL */
8557
8558 if (TRPMHasTrap(pVCpu))
8559 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8560 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8561
8562 /*
8563 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8564 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8565 */
8566 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8567 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8568 { /* likely */ }
8569 else
8570 {
8571 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8572 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8573 return rcStrict;
8574 }
8575
8576 /*
8577 * Load the guest state bits, we can handle longjmps/getting preempted here.
8578 *
8579 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8580 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8581 * Hence, this needs to be done -after- injection of events.
8582 */
8583 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8584 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8585 { /* likely */ }
8586 else
8587 return rcStrict;
8588
8589 /*
8590 * No longjmps to ring-3 from this point on!!!
8591 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8592 * This also disables flushing of the R0-logger instance (if any).
8593 */
8594 VMMRZCallRing3Disable(pVCpu);
8595
8596 /*
8597 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8598 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8599 *
8600 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8601 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8602 *
8603 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8604 * executing guest code.
8605 */
8606 pVmxTransient->fEFlags = ASMIntDisableFlags();
8607
8608 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8609 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8610 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8611 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8612 {
8613 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8614 {
8615 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8616 pVCpu->hm.s.Event.fPending = false;
8617
8618 return VINF_SUCCESS;
8619 }
8620
8621 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8622 rcStrict = VINF_EM_RAW_INTERRUPT;
8623 }
8624 else
8625 {
8626 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8627 rcStrict = VINF_EM_RAW_TO_R3;
8628 }
8629
8630 ASMSetFlags(pVmxTransient->fEFlags);
8631 VMMRZCallRing3Enable(pVCpu);
8632
8633 return rcStrict;
8634}
8635
8636
8637/**
8638 * Prepares to run guest code in VT-x and we've committed to doing so. This
8639 * means there is no backing out to ring-3 or anywhere else at this
8640 * point.
8641 *
8642 * @param pVM The cross context VM structure.
8643 * @param pVCpu The cross context virtual CPU structure.
8644 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8645 * out-of-sync. Make sure to update the required fields
8646 * before using them.
8647 * @param pVmxTransient Pointer to the VMX transient structure.
8648 *
8649 * @remarks Called with preemption disabled.
8650 * @remarks No-long-jump zone!!!
8651 */
8652static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8653{
8654 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8655 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8656 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8657
8658 /*
8659 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8660 */
8661 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8662 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8663
8664#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8665 if (!CPUMIsGuestFPUStateActive(pVCpu))
8666 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8667 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8668 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8669#endif
8670
8671 if ( pVCpu->hm.s.fPreloadGuestFpu
8672 && !CPUMIsGuestFPUStateActive(pVCpu))
8673 {
8674 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8675 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8676 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8677 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8678 }
8679
8680 /*
8681 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8682 */
8683 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8684 && pVCpu->hm.s.vmx.cMsrs > 0)
8685 {
8686 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8687 }
8688
8689 /*
8690 * Load the host state bits as we may've been preempted (only happens when
8691 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8692 */
8693 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8694 * any effect to the host state needing to be saved? */
8695 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8696 {
8697 /* This ASSUMES that pfnStartVM has been set up already. */
8698 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8699 AssertRC(rc);
8700 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8701 }
8702 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8703
8704 /*
8705 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8706 */
8707 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8708 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8709 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8710
8711 /* Store status of the shared guest-host state at the time of VM-entry. */
8712#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8713 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8714 {
8715 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8716 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8717 }
8718 else
8719#endif
8720 {
8721 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8722 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8723 }
8724 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8725
8726 /*
8727 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8728 */
8729 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8730 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8731
8732 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8733 RTCPUID idCurrentCpu = pCpu->idCpu;
8734 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8735 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8736 {
8737 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8738 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8739 }
8740
8741 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8742 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8743 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8744 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8745
8746 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8747
8748 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8749 to start executing. */
8750
8751 /*
8752 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8753 */
8754 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8755 {
8756 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8757 {
8758 bool fMsrUpdated;
8759 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8760 AssertRC(rc2);
8761 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8762
8763 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8764 &fMsrUpdated);
8765 AssertRC(rc2);
8766 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8767
8768 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8769 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8770 }
8771 else
8772 {
8773 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8774 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8775 }
8776 }
8777
8778#ifdef VBOX_STRICT
8779 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8780 hmR0VmxCheckHostEferMsr(pVCpu);
8781 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8782#endif
8783#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8784 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8785 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8786 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8787#endif
8788}
8789
8790
8791/**
8792 * Performs some essential restoration of state after running guest code in
8793 * VT-x.
8794 *
8795 * @param pVM The cross context VM structure.
8796 * @param pVCpu The cross context virtual CPU structure.
8797 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8798 * out-of-sync. Make sure to update the required fields
8799 * before using them.
8800 * @param pVmxTransient Pointer to the VMX transient structure.
8801 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8802 *
8803 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8804 *
8805 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8806 * unconditionally when it is safe to do so.
8807 */
8808static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8809{
8810 NOREF(pVM);
8811
8812 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8813
8814 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8815 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8816 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8817 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8818 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8819 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8820
8821 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8822 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8823
8824 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8825 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8826 Assert(!ASMIntAreEnabled());
8827 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8828
8829#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8830 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8831 {
8832 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8833 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8834 }
8835#endif
8836
8837#if HC_ARCH_BITS == 64
8838 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8839#endif
8840 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8841#ifdef VBOX_STRICT
8842 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8843#endif
8844 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8845 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8846
8847 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8848 uint32_t uExitReason;
8849 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8850 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8851 AssertRC(rc);
8852 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8853 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8854
8855 /* Update the VM-exit history array. */
8856 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8857
8858 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8859 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8860 {
8861 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8862 pVmxTransient->fVMEntryFailed));
8863 return;
8864 }
8865
8866 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8867 {
8868 /** @todo We can optimize this by only syncing with our force-flags when
8869 * really needed and keeping the VMCS state as it is for most
8870 * VM-exits. */
8871 /* Update the guest interruptibility-state from the VMCS. */
8872 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8873
8874#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8875 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8876 AssertRC(rc);
8877#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8878 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8879 AssertRC(rc);
8880#endif
8881
8882 /*
8883 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8884 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8885 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8886 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8887 */
8888 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8889 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8890 {
8891 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8892 AssertRC(rc);
8893 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8894 }
8895 }
8896}
8897
8898
8899/**
8900 * Runs the guest code using VT-x the normal way.
8901 *
8902 * @returns VBox status code.
8903 * @param pVM The cross context VM structure.
8904 * @param pVCpu The cross context virtual CPU structure.
8905 * @param pCtx Pointer to the guest-CPU context.
8906 *
8907 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8908 */
8909static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8910{
8911 VMXTRANSIENT VmxTransient;
8912 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8913 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8914 uint32_t cLoops = 0;
8915
8916 for (;; cLoops++)
8917 {
8918 Assert(!HMR0SuspendPending());
8919 HMVMX_ASSERT_CPU_SAFE();
8920
8921 /* Preparatory work for running guest code, this may force us to return
8922 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8923 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8924 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8925 if (rcStrict != VINF_SUCCESS)
8926 break;
8927
8928 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8929 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8930 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8931
8932 /* Restore any residual host-state and save any bits shared between host
8933 and guest into the guest-CPU state. Re-enables interrupts! */
8934 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8935
8936 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8937 if (RT_SUCCESS(rcRun))
8938 { /* very likely */ }
8939 else
8940 {
8941 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8942 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8943 return rcRun;
8944 }
8945
8946 /* Profile the VM-exit. */
8947 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8948 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8949 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8950 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8951 HMVMX_START_EXIT_DISPATCH_PROF();
8952
8953 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8954
8955 /* Handle the VM-exit. */
8956#ifdef HMVMX_USE_FUNCTION_TABLE
8957 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8958#else
8959 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8960#endif
8961 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8962 if (rcStrict == VINF_SUCCESS)
8963 {
8964 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8965 continue; /* likely */
8966 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8967 rcStrict = VINF_EM_RAW_INTERRUPT;
8968 }
8969 break;
8970 }
8971
8972 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8973 return rcStrict;
8974}
8975
8976
8977
8978/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8979 * probes.
8980 *
8981 * The following few functions and associated structure contains the bloat
8982 * necessary for providing detailed debug events and dtrace probes as well as
8983 * reliable host side single stepping. This works on the principle of
8984 * "subclassing" the normal execution loop and workers. We replace the loop
8985 * method completely and override selected helpers to add necessary adjustments
8986 * to their core operation.
8987 *
8988 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8989 * any performance for debug and analysis features.
8990 *
8991 * @{
8992 */
8993
8994typedef struct VMXRUNDBGSTATE
8995{
8996 /** The RIP we started executing at. This is for detecting that we stepped. */
8997 uint64_t uRipStart;
8998 /** The CS we started executing with. */
8999 uint16_t uCsStart;
9000
9001 /** Whether we've actually modified the 1st execution control field. */
9002 bool fModifiedProcCtls : 1;
9003 /** Whether we've actually modified the 2nd execution control field. */
9004 bool fModifiedProcCtls2 : 1;
9005 /** Whether we've actually modified the exception bitmap. */
9006 bool fModifiedXcptBitmap : 1;
9007
9008 /** We desire the modified the CR0 mask to be cleared. */
9009 bool fClearCr0Mask : 1;
9010 /** We desire the modified the CR4 mask to be cleared. */
9011 bool fClearCr4Mask : 1;
9012 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9013 uint32_t fCpe1Extra;
9014 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9015 uint32_t fCpe1Unwanted;
9016 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9017 uint32_t fCpe2Extra;
9018 /** Extra stuff we need in */
9019 uint32_t bmXcptExtra;
9020 /** The sequence number of the Dtrace provider settings the state was
9021 * configured against. */
9022 uint32_t uDtraceSettingsSeqNo;
9023 /** Exits to check (one bit per exit). */
9024 uint32_t bmExitsToCheck[3];
9025
9026 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9027 uint32_t fProcCtlsInitial;
9028 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9029 uint32_t fProcCtls2Initial;
9030 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9031 uint32_t bmXcptInitial;
9032} VMXRUNDBGSTATE;
9033AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9034typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9035
9036
9037/**
9038 * Initializes the VMXRUNDBGSTATE structure.
9039 *
9040 * @param pVCpu The cross context virtual CPU structure of the
9041 * calling EMT.
9042 * @param pCtx The CPU register context to go with @a pVCpu.
9043 * @param pDbgState The structure to initialize.
9044 */
9045DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9046{
9047 pDbgState->uRipStart = pCtx->rip;
9048 pDbgState->uCsStart = pCtx->cs.Sel;
9049
9050 pDbgState->fModifiedProcCtls = false;
9051 pDbgState->fModifiedProcCtls2 = false;
9052 pDbgState->fModifiedXcptBitmap = false;
9053 pDbgState->fClearCr0Mask = false;
9054 pDbgState->fClearCr4Mask = false;
9055 pDbgState->fCpe1Extra = 0;
9056 pDbgState->fCpe1Unwanted = 0;
9057 pDbgState->fCpe2Extra = 0;
9058 pDbgState->bmXcptExtra = 0;
9059 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9060 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9061 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9062}
9063
9064
9065/**
9066 * Updates the VMSC fields with changes requested by @a pDbgState.
9067 *
9068 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9069 * immediately before executing guest code, i.e. when interrupts are disabled.
9070 * We don't check status codes here as we cannot easily assert or return in the
9071 * latter case.
9072 *
9073 * @param pVCpu The cross context virtual CPU structure.
9074 * @param pDbgState The debug state.
9075 */
9076DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9077{
9078 /*
9079 * Ensure desired flags in VMCS control fields are set.
9080 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9081 *
9082 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9083 * there should be no stale data in pCtx at this point.
9084 */
9085 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9086 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9087 {
9088 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9089 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9090 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9091 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9092 pDbgState->fModifiedProcCtls = true;
9093 }
9094
9095 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9096 {
9097 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9098 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9099 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9100 pDbgState->fModifiedProcCtls2 = true;
9101 }
9102
9103 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9104 {
9105 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9106 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9107 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9108 pDbgState->fModifiedXcptBitmap = true;
9109 }
9110
9111 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9112 {
9113 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9114 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9115 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9116 }
9117
9118 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9119 {
9120 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9121 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9122 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9123 }
9124}
9125
9126
9127DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9128{
9129 /*
9130 * Restore exit control settings as we may not reenter this function the
9131 * next time around.
9132 */
9133 /* We reload the initial value, trigger what we can of recalculations the
9134 next time around. From the looks of things, that's all that's required atm. */
9135 if (pDbgState->fModifiedProcCtls)
9136 {
9137 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9138 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9139 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9140 AssertRCReturn(rc2, rc2);
9141 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9142 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9143 }
9144
9145 /* We're currently the only ones messing with this one, so just restore the
9146 cached value and reload the field. */
9147 if ( pDbgState->fModifiedProcCtls2
9148 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9149 {
9150 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9151 AssertRCReturn(rc2, rc2);
9152 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9153 }
9154
9155 /* If we've modified the exception bitmap, we restore it and trigger
9156 reloading and partial recalculation the next time around. */
9157 if (pDbgState->fModifiedXcptBitmap)
9158 {
9159 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9160 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9161 }
9162
9163 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9164 if (pDbgState->fClearCr0Mask)
9165 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9166
9167 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9168 if (pDbgState->fClearCr4Mask)
9169 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9170
9171 return rcStrict;
9172}
9173
9174
9175/**
9176 * Configures VM-exit controls for current DBGF and DTrace settings.
9177 *
9178 * This updates @a pDbgState and the VMCS execution control fields to reflect
9179 * the necessary exits demanded by DBGF and DTrace.
9180 *
9181 * @param pVM The cross context VM structure.
9182 * @param pVCpu The cross context virtual CPU structure.
9183 * @param pCtx Pointer to the guest-CPU context.
9184 * @param pDbgState The debug state.
9185 * @param pVmxTransient Pointer to the VMX transient structure. May update
9186 * fUpdateTscOffsettingAndPreemptTimer.
9187 */
9188static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9189 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9190{
9191 /*
9192 * Take down the dtrace serial number so we can spot changes.
9193 */
9194 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9195 ASMCompilerBarrier();
9196
9197 /*
9198 * We'll rebuild most of the middle block of data members (holding the
9199 * current settings) as we go along here, so start by clearing it all.
9200 */
9201 pDbgState->bmXcptExtra = 0;
9202 pDbgState->fCpe1Extra = 0;
9203 pDbgState->fCpe1Unwanted = 0;
9204 pDbgState->fCpe2Extra = 0;
9205 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9206 pDbgState->bmExitsToCheck[i] = 0;
9207
9208 /*
9209 * Software interrupts (INT XXh) - no idea how to trigger these...
9210 */
9211 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9212 || VBOXVMM_INT_SOFTWARE_ENABLED())
9213 {
9214 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9215 }
9216
9217 /*
9218 * Exception bitmap and XCPT events+probes.
9219 */
9220 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9221 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9222 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9223
9224 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9225 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9226 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9227 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9228 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9229 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9230 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9231 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9232 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9233 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9234 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9235 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9236 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9237 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9238 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9239 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9240 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9241 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9242
9243 if (pDbgState->bmXcptExtra)
9244 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9245
9246 /*
9247 * Process events and probes for VM exits, making sure we get the wanted exits.
9248 *
9249 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9250 * So, when adding/changing/removing please don't forget to update it.
9251 *
9252 * Some of the macros are picking up local variables to save horizontal space,
9253 * (being able to see it in a table is the lesser evil here).
9254 */
9255#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9256 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9257 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9258#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9259 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9260 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9261 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9262 } else do { } while (0)
9263#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9264 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9265 { \
9266 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9267 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9268 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9269 } else do { } while (0)
9270#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9271 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9272 { \
9273 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9274 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9275 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9276 } else do { } while (0)
9277#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9278 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9279 { \
9280 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9281 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9282 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9283 } else do { } while (0)
9284
9285 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9286 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9287 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9288 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9289 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9290
9291 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9292 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9293 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9294 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9295 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9296 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9297 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9298 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9299 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9300 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9301 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9302 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9303 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9304 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9305 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9306 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9307 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9308 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9309 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9310 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9311 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9312 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9313 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9314 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9315 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9316 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9317 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9318 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9319 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9320 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9321 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9322 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9323 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9324 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9325 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9326 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9327
9328 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9329 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9330 {
9331 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9332 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9333 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9334 AssertRC(rc2);
9335
9336#if 0 /** @todo fix me */
9337 pDbgState->fClearCr0Mask = true;
9338 pDbgState->fClearCr4Mask = true;
9339#endif
9340 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9341 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9342 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9343 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9344 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9345 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9346 require clearing here and in the loop if we start using it. */
9347 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9348 }
9349 else
9350 {
9351 if (pDbgState->fClearCr0Mask)
9352 {
9353 pDbgState->fClearCr0Mask = false;
9354 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9355 }
9356 if (pDbgState->fClearCr4Mask)
9357 {
9358 pDbgState->fClearCr4Mask = false;
9359 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9360 }
9361 }
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9363 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9364
9365 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9366 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9367 {
9368 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9369 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9370 }
9371 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9372 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9373
9374 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9375 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9376 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9377 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9378 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9379 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9380 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9382#if 0 /** @todo too slow, fix handler. */
9383 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9384#endif
9385 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9386
9387 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9388 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9389 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9390 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9391 {
9392 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9393 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9394 }
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9397 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9398 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9399
9400 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9401 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9402 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9403 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9404 {
9405 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9406 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9407 }
9408 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9410 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9412
9413 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9414 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9415 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9416 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9417 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9418 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9419 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9420 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9421 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9422 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9423 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9424 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9425 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9426 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9427 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9428 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9429 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9430 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9431 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9432 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9433 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9434 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9435
9436#undef IS_EITHER_ENABLED
9437#undef SET_ONLY_XBM_IF_EITHER_EN
9438#undef SET_CPE1_XBM_IF_EITHER_EN
9439#undef SET_CPEU_XBM_IF_EITHER_EN
9440#undef SET_CPE2_XBM_IF_EITHER_EN
9441
9442 /*
9443 * Sanitize the control stuff.
9444 */
9445 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9446 if (pDbgState->fCpe2Extra)
9447 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9448 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9449 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9450 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9451 {
9452 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9453 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9454 }
9455
9456 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9457 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9458 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9459 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9460}
9461
9462
9463/**
9464 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9465 *
9466 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9467 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9468 * either.
9469 *
9470 * @returns Strict VBox status code (i.e. informational status codes too).
9471 * @param pVM The cross context VM structure.
9472 * @param pVCpu The cross context virtual CPU structure.
9473 * @param pMixedCtx Pointer to the guest-CPU context.
9474 * @param pVmxTransient Pointer to the VMX-transient structure.
9475 * @param uExitReason The VM-exit reason.
9476 *
9477 * @remarks The name of this function is displayed by dtrace, so keep it short
9478 * and to the point. No longer than 33 chars long, please.
9479 */
9480static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9481 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9482{
9483 /*
9484 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9485 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9486 *
9487 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9488 * does. Must add/change/remove both places. Same ordering, please.
9489 *
9490 * Added/removed events must also be reflected in the next section
9491 * where we dispatch dtrace events.
9492 */
9493 bool fDtrace1 = false;
9494 bool fDtrace2 = false;
9495 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9496 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9497 uint32_t uEventArg = 0;
9498#define SET_EXIT(a_EventSubName) \
9499 do { \
9500 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9501 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9502 } while (0)
9503#define SET_BOTH(a_EventSubName) \
9504 do { \
9505 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9506 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9507 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9508 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9509 } while (0)
9510 switch (uExitReason)
9511 {
9512 case VMX_EXIT_MTF:
9513 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9514
9515 case VMX_EXIT_XCPT_OR_NMI:
9516 {
9517 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9518 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9519 {
9520 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9521 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9522 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9523 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9524 {
9525 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9526 {
9527 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9528 uEventArg = pVmxTransient->uExitIntErrorCode;
9529 }
9530 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9531 switch (enmEvent1)
9532 {
9533 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9534 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9535 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9536 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9537 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9538 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9539 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9540 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9541 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9542 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9543 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9544 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9545 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9546 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9547 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9548 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9549 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9550 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9551 default: break;
9552 }
9553 }
9554 else
9555 AssertFailed();
9556 break;
9557
9558 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9559 uEventArg = idxVector;
9560 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9561 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9562 break;
9563 }
9564 break;
9565 }
9566
9567 case VMX_EXIT_TRIPLE_FAULT:
9568 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9569 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9570 break;
9571 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9572 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9573 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9574 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9575 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9576
9577 /* Instruction specific VM-exits: */
9578 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9579 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9580 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9581 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9582 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9583 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9584 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9585 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9586 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9587 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9588 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9589 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9590 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9591 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9592 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9593 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9594 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9595 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9596 case VMX_EXIT_MOV_CRX:
9597 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9598/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9599* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9600 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9601 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9602 SET_BOTH(CRX_READ);
9603 else
9604 SET_BOTH(CRX_WRITE);
9605 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9606 break;
9607 case VMX_EXIT_MOV_DRX:
9608 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9609 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9610 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9611 SET_BOTH(DRX_READ);
9612 else
9613 SET_BOTH(DRX_WRITE);
9614 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9615 break;
9616 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9617 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9618 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9619 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9620 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9621 case VMX_EXIT_XDTR_ACCESS:
9622 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9623 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9624 {
9625 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9626 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9627 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9628 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9629 }
9630 break;
9631
9632 case VMX_EXIT_TR_ACCESS:
9633 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9634 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9635 {
9636 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9637 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9638 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9639 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9640 }
9641 break;
9642
9643 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9644 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9645 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9646 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9647 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9648 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9649 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9650 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9651 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9652 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9653 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9654
9655 /* Events that aren't relevant at this point. */
9656 case VMX_EXIT_EXT_INT:
9657 case VMX_EXIT_INT_WINDOW:
9658 case VMX_EXIT_NMI_WINDOW:
9659 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9660 case VMX_EXIT_PREEMPT_TIMER:
9661 case VMX_EXIT_IO_INSTR:
9662 break;
9663
9664 /* Errors and unexpected events. */
9665 case VMX_EXIT_INIT_SIGNAL:
9666 case VMX_EXIT_SIPI:
9667 case VMX_EXIT_IO_SMI:
9668 case VMX_EXIT_SMI:
9669 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9670 case VMX_EXIT_ERR_MSR_LOAD:
9671 case VMX_EXIT_ERR_MACHINE_CHECK:
9672 break;
9673
9674 default:
9675 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9676 break;
9677 }
9678#undef SET_BOTH
9679#undef SET_EXIT
9680
9681 /*
9682 * Dtrace tracepoints go first. We do them here at once so we don't
9683 * have to copy the guest state saving and stuff a few dozen times.
9684 * Down side is that we've got to repeat the switch, though this time
9685 * we use enmEvent since the probes are a subset of what DBGF does.
9686 */
9687 if (fDtrace1 || fDtrace2)
9688 {
9689 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9690 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9691 switch (enmEvent1)
9692 {
9693 /** @todo consider which extra parameters would be helpful for each probe. */
9694 case DBGFEVENT_END: break;
9695 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9696 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9697 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9698 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9699 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9700 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9701 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9703 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9704 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9705 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9706 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9707 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9708 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9709 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9710 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9711 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9712 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9713 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9714 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9715 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9716 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9717 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9718 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9719 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9720 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9721 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9722 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9723 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9724 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9725 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9726 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9727 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9728 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9729 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9730 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9731 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9732 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9733 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9734 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9735 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9736 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9737 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9738 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9739 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9740 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9741 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9742 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9743 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9744 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9745 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9746 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9747 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9748 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9749 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9750 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9751 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9752 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9754 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9760 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9761 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9762 }
9763 switch (enmEvent2)
9764 {
9765 /** @todo consider which extra parameters would be helpful for each probe. */
9766 case DBGFEVENT_END: break;
9767 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9768 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9769 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9770 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9771 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9772 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9777 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9778 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9779 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9780 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9781 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9782 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9783 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9784 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9804 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9805 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9806 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9807 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9808 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9809 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9810 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9811 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9812 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9813 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9814 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9815 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9816 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9817 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9818 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9819 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9820 }
9821 }
9822
9823 /*
9824 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9825 * the DBGF call will do a full check).
9826 *
9827 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9828 * Note! If we have to events, we prioritize the first, i.e. the instruction
9829 * one, in order to avoid event nesting.
9830 */
9831 if ( enmEvent1 != DBGFEVENT_END
9832 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9833 {
9834 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9835 if (rcStrict != VINF_SUCCESS)
9836 return rcStrict;
9837 }
9838 else if ( enmEvent2 != DBGFEVENT_END
9839 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9840 {
9841 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9842 if (rcStrict != VINF_SUCCESS)
9843 return rcStrict;
9844 }
9845
9846 return VINF_SUCCESS;
9847}
9848
9849
9850/**
9851 * Single-stepping VM-exit filtering.
9852 *
9853 * This is preprocessing the exits and deciding whether we've gotten far enough
9854 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9855 * performed.
9856 *
9857 * @returns Strict VBox status code (i.e. informational status codes too).
9858 * @param pVM The cross context VM structure.
9859 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9860 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9861 * out-of-sync. Make sure to update the required
9862 * fields before using them.
9863 * @param pVmxTransient Pointer to the VMX-transient structure.
9864 * @param uExitReason The VM-exit reason.
9865 * @param pDbgState The debug state.
9866 */
9867DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9868 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9869{
9870 /*
9871 * Expensive (saves context) generic dtrace exit probe.
9872 */
9873 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9874 { /* more likely */ }
9875 else
9876 {
9877 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9878 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9879 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9880 }
9881
9882 /*
9883 * Check for host NMI, just to get that out of the way.
9884 */
9885 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9886 { /* normally likely */ }
9887 else
9888 {
9889 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9890 AssertRCReturn(rc2, rc2);
9891 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9892 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9893 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9894 }
9895
9896 /*
9897 * Check for single stepping event if we're stepping.
9898 */
9899 if (pVCpu->hm.s.fSingleInstruction)
9900 {
9901 switch (uExitReason)
9902 {
9903 case VMX_EXIT_MTF:
9904 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9905
9906 /* Various events: */
9907 case VMX_EXIT_XCPT_OR_NMI:
9908 case VMX_EXIT_EXT_INT:
9909 case VMX_EXIT_TRIPLE_FAULT:
9910 case VMX_EXIT_INT_WINDOW:
9911 case VMX_EXIT_NMI_WINDOW:
9912 case VMX_EXIT_TASK_SWITCH:
9913 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9914 case VMX_EXIT_APIC_ACCESS:
9915 case VMX_EXIT_EPT_VIOLATION:
9916 case VMX_EXIT_EPT_MISCONFIG:
9917 case VMX_EXIT_PREEMPT_TIMER:
9918
9919 /* Instruction specific VM-exits: */
9920 case VMX_EXIT_CPUID:
9921 case VMX_EXIT_GETSEC:
9922 case VMX_EXIT_HLT:
9923 case VMX_EXIT_INVD:
9924 case VMX_EXIT_INVLPG:
9925 case VMX_EXIT_RDPMC:
9926 case VMX_EXIT_RDTSC:
9927 case VMX_EXIT_RSM:
9928 case VMX_EXIT_VMCALL:
9929 case VMX_EXIT_VMCLEAR:
9930 case VMX_EXIT_VMLAUNCH:
9931 case VMX_EXIT_VMPTRLD:
9932 case VMX_EXIT_VMPTRST:
9933 case VMX_EXIT_VMREAD:
9934 case VMX_EXIT_VMRESUME:
9935 case VMX_EXIT_VMWRITE:
9936 case VMX_EXIT_VMXOFF:
9937 case VMX_EXIT_VMXON:
9938 case VMX_EXIT_MOV_CRX:
9939 case VMX_EXIT_MOV_DRX:
9940 case VMX_EXIT_IO_INSTR:
9941 case VMX_EXIT_RDMSR:
9942 case VMX_EXIT_WRMSR:
9943 case VMX_EXIT_MWAIT:
9944 case VMX_EXIT_MONITOR:
9945 case VMX_EXIT_PAUSE:
9946 case VMX_EXIT_XDTR_ACCESS:
9947 case VMX_EXIT_TR_ACCESS:
9948 case VMX_EXIT_INVEPT:
9949 case VMX_EXIT_RDTSCP:
9950 case VMX_EXIT_INVVPID:
9951 case VMX_EXIT_WBINVD:
9952 case VMX_EXIT_XSETBV:
9953 case VMX_EXIT_RDRAND:
9954 case VMX_EXIT_INVPCID:
9955 case VMX_EXIT_VMFUNC:
9956 case VMX_EXIT_RDSEED:
9957 case VMX_EXIT_XSAVES:
9958 case VMX_EXIT_XRSTORS:
9959 {
9960 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9961 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9962 AssertRCReturn(rc2, rc2);
9963 if ( pMixedCtx->rip != pDbgState->uRipStart
9964 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9965 return VINF_EM_DBG_STEPPED;
9966 break;
9967 }
9968
9969 /* Errors and unexpected events: */
9970 case VMX_EXIT_INIT_SIGNAL:
9971 case VMX_EXIT_SIPI:
9972 case VMX_EXIT_IO_SMI:
9973 case VMX_EXIT_SMI:
9974 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9975 case VMX_EXIT_ERR_MSR_LOAD:
9976 case VMX_EXIT_ERR_MACHINE_CHECK:
9977 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9978 break;
9979
9980 default:
9981 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9982 break;
9983 }
9984 }
9985
9986 /*
9987 * Check for debugger event breakpoints and dtrace probes.
9988 */
9989 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9990 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9991 {
9992 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9993 if (rcStrict != VINF_SUCCESS)
9994 return rcStrict;
9995 }
9996
9997 /*
9998 * Normal processing.
9999 */
10000#ifdef HMVMX_USE_FUNCTION_TABLE
10001 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10002#else
10003 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10004#endif
10005}
10006
10007
10008/**
10009 * Single steps guest code using VT-x.
10010 *
10011 * @returns Strict VBox status code (i.e. informational status codes too).
10012 * @param pVM The cross context VM structure.
10013 * @param pVCpu The cross context virtual CPU structure.
10014 * @param pCtx Pointer to the guest-CPU context.
10015 *
10016 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10017 */
10018static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10019{
10020 VMXTRANSIENT VmxTransient;
10021 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10022
10023 /* Set HMCPU indicators. */
10024 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10025 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10026 pVCpu->hm.s.fDebugWantRdTscExit = false;
10027 pVCpu->hm.s.fUsingDebugLoop = true;
10028
10029 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10030 VMXRUNDBGSTATE DbgState;
10031 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10032 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10033
10034 /*
10035 * The loop.
10036 */
10037 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10038 for (uint32_t cLoops = 0; ; cLoops++)
10039 {
10040 Assert(!HMR0SuspendPending());
10041 HMVMX_ASSERT_CPU_SAFE();
10042 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10043
10044 /*
10045 * Preparatory work for running guest code, this may force us to return
10046 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10047 */
10048 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10049 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10050 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10051 if (rcStrict != VINF_SUCCESS)
10052 break;
10053
10054 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10055 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10056
10057 /*
10058 * Now we can run the guest code.
10059 */
10060 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10061
10062 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10063
10064 /*
10065 * Restore any residual host-state and save any bits shared between host
10066 * and guest into the guest-CPU state. Re-enables interrupts!
10067 */
10068 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10069
10070 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10071 if (RT_SUCCESS(rcRun))
10072 { /* very likely */ }
10073 else
10074 {
10075 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10076 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10077 return rcRun;
10078 }
10079
10080 /* Profile the VM-exit. */
10081 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10082 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10083 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10084 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10085 HMVMX_START_EXIT_DISPATCH_PROF();
10086
10087 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10088
10089 /*
10090 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10091 */
10092 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10093 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10094 if (rcStrict != VINF_SUCCESS)
10095 break;
10096 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10097 {
10098 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10099 rcStrict = VINF_EM_RAW_INTERRUPT;
10100 break;
10101 }
10102
10103 /*
10104 * Stepping: Did the RIP change, if so, consider it a single step.
10105 * Otherwise, make sure one of the TFs gets set.
10106 */
10107 if (fStepping)
10108 {
10109 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10110 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10111 AssertRCReturn(rc2, rc2);
10112 if ( pCtx->rip != DbgState.uRipStart
10113 || pCtx->cs.Sel != DbgState.uCsStart)
10114 {
10115 rcStrict = VINF_EM_DBG_STEPPED;
10116 break;
10117 }
10118 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10119 }
10120
10121 /*
10122 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10123 */
10124 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10125 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10126 }
10127
10128 /*
10129 * Clear the X86_EFL_TF if necessary.
10130 */
10131 if (pVCpu->hm.s.fClearTrapFlag)
10132 {
10133 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10134 AssertRCReturn(rc2, rc2);
10135 pVCpu->hm.s.fClearTrapFlag = false;
10136 pCtx->eflags.Bits.u1TF = 0;
10137 }
10138 /** @todo there seems to be issues with the resume flag when the monitor trap
10139 * flag is pending without being used. Seen early in bios init when
10140 * accessing APIC page in protected mode. */
10141
10142 /*
10143 * Restore VM-exit control settings as we may not reenter this function the
10144 * next time around.
10145 */
10146 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10147
10148 /* Restore HMCPU indicators. */
10149 pVCpu->hm.s.fUsingDebugLoop = false;
10150 pVCpu->hm.s.fDebugWantRdTscExit = false;
10151 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10152
10153 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10154 return rcStrict;
10155}
10156
10157
10158/** @} */
10159
10160
10161/**
10162 * Checks if any expensive dtrace probes are enabled and we should go to the
10163 * debug loop.
10164 *
10165 * @returns true if we should use debug loop, false if not.
10166 */
10167static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10168{
10169 /* It's probably faster to OR the raw 32-bit counter variables together.
10170 Since the variables are in an array and the probes are next to one
10171 another (more or less), we have good locality. So, better read
10172 eight-nine cache lines ever time and only have one conditional, than
10173 128+ conditionals, right? */
10174 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10175 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10176 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10177 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10178 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10179 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10180 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10181 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10182 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10183 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10184 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10185 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10186 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10187 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10188 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10189 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10190 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10191 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10192 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10193 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10194 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10195 ) != 0
10196 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10197 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10198 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10199 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10200 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10201 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10202 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10203 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10204 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10205 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10206 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10207 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10208 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10209 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10210 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10211 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10212 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10213 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10214 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10215 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10216 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10217 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10218 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10219 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10220 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10221 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10222 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10223 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10224 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10225 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10226 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10227 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10228 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10229 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10230 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10231 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10232 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10233 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10234 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10235 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10236 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10237 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10238 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10239 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10240 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10241 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10242 ) != 0
10243 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10244 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10245 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10246 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10247 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10248 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10249 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10250 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10251 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10252 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10253 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10254 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10255 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10256 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10257 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10258 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10259 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10260 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10261 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10262 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10263 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10264 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10265 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10266 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10267 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10268 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10269 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10270 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10271 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10272 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10273 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10274 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10275 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10279 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10280 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10281 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10282 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10283 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10284 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10285 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10286 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10287 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10288 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10289 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10290 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10291 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10292 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10293 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10294 ) != 0;
10295}
10296
10297
10298/**
10299 * Runs the guest code using VT-x.
10300 *
10301 * @returns Strict VBox status code (i.e. informational status codes too).
10302 * @param pVM The cross context VM structure.
10303 * @param pVCpu The cross context virtual CPU structure.
10304 * @param pCtx Pointer to the guest-CPU context.
10305 */
10306VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10307{
10308 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10309 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10310 HMVMX_ASSERT_PREEMPT_SAFE();
10311
10312 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10313
10314 VBOXSTRICTRC rcStrict;
10315 if ( !pVCpu->hm.s.fUseDebugLoop
10316 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10317 && !DBGFIsStepping(pVCpu) )
10318 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10319 else
10320 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10321
10322 if (rcStrict == VERR_EM_INTERPRETER)
10323 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10324 else if (rcStrict == VINF_EM_RESET)
10325 rcStrict = VINF_EM_TRIPLE_FAULT;
10326
10327 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10328 if (RT_FAILURE(rc2))
10329 {
10330 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10331 rcStrict = rc2;
10332 }
10333 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10334 return rcStrict;
10335}
10336
10337
10338#ifndef HMVMX_USE_FUNCTION_TABLE
10339DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10340{
10341# ifdef DEBUG_ramshankar
10342# define RETURN_EXIT_CALL(a_CallExpr) \
10343 do { \
10344 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10345 VBOXSTRICTRC rcStrict = a_CallExpr; \
10346 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10347 return rcStrict; \
10348 } while (0)
10349# else
10350# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10351# endif
10352 switch (rcReason)
10353 {
10354 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10355 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10356 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10357 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10358 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10359 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10360 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10361 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10362 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10363 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10364 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10365 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10366 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10367 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10368 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10369 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10370 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10371 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10372 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10373 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10374 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10375 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10376 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10377 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10378 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10379 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10380 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10381 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10382 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10383 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10384 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10385 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10386 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10387 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10388
10389 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10390 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10391 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10392 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10393 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10394 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10395 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10396 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10397 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10398
10399 case VMX_EXIT_VMCLEAR:
10400 case VMX_EXIT_VMLAUNCH:
10401 case VMX_EXIT_VMPTRLD:
10402 case VMX_EXIT_VMPTRST:
10403 case VMX_EXIT_VMREAD:
10404 case VMX_EXIT_VMRESUME:
10405 case VMX_EXIT_VMWRITE:
10406 case VMX_EXIT_VMXOFF:
10407 case VMX_EXIT_VMXON:
10408 case VMX_EXIT_INVEPT:
10409 case VMX_EXIT_INVVPID:
10410 case VMX_EXIT_VMFUNC:
10411 case VMX_EXIT_XSAVES:
10412 case VMX_EXIT_XRSTORS:
10413 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10414 case VMX_EXIT_RESERVED_60:
10415 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10416 case VMX_EXIT_RESERVED_62:
10417 default:
10418 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10419 }
10420#undef RETURN_EXIT_CALL
10421}
10422#endif /* !HMVMX_USE_FUNCTION_TABLE */
10423
10424
10425#ifdef VBOX_STRICT
10426/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10427# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10428 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10429
10430# define HMVMX_ASSERT_PREEMPT_CPUID() \
10431 do { \
10432 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10433 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10434 } while (0)
10435
10436# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10437 do { \
10438 AssertPtr(pVCpu); \
10439 AssertPtr(pMixedCtx); \
10440 AssertPtr(pVmxTransient); \
10441 Assert(pVmxTransient->fVMEntryFailed == false); \
10442 Assert(ASMIntAreEnabled()); \
10443 HMVMX_ASSERT_PREEMPT_SAFE(); \
10444 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10445 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10446 HMVMX_ASSERT_PREEMPT_SAFE(); \
10447 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10448 HMVMX_ASSERT_PREEMPT_CPUID(); \
10449 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10450 } while (0)
10451
10452# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10453 do { \
10454 Log4Func(("\n")); \
10455 } while (0)
10456#else /* nonstrict builds: */
10457# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10458 do { \
10459 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10460 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10461 } while (0)
10462# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10463#endif
10464
10465
10466/**
10467 * Advances the guest RIP by the specified number of bytes.
10468 *
10469 * @param pVCpu The cross context virtual CPU structure.
10470 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10471 * out-of-sync. Make sure to update the required fields
10472 * before using them.
10473 * @param cbInstr Number of bytes to advance the RIP by.
10474 *
10475 * @remarks No-long-jump zone!!!
10476 */
10477DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10478{
10479 /* Advance the RIP. */
10480 pMixedCtx->rip += cbInstr;
10481 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10482
10483 /* Update interrupt inhibition. */
10484 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10485 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10486 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10487}
10488
10489
10490/**
10491 * Advances the guest RIP after reading it from the VMCS.
10492 *
10493 * @returns VBox status code, no informational status codes.
10494 * @param pVCpu The cross context virtual CPU structure.
10495 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10496 * out-of-sync. Make sure to update the required fields
10497 * before using them.
10498 * @param pVmxTransient Pointer to the VMX transient structure.
10499 *
10500 * @remarks No-long-jump zone!!!
10501 */
10502static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10503{
10504 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10505 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10506 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10507 AssertRCReturn(rc, rc);
10508
10509 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10510
10511 /*
10512 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10513 * pending debug exception field as it takes care of priority of events.
10514 *
10515 * See Intel spec. 32.2.1 "Debug Exceptions".
10516 */
10517 if ( !pVCpu->hm.s.fSingleInstruction
10518 && pMixedCtx->eflags.Bits.u1TF)
10519 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10520
10521 return VINF_SUCCESS;
10522}
10523
10524
10525/**
10526 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10527 * and update error record fields accordingly.
10528 *
10529 * @return VMX_IGS_* return codes.
10530 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10531 * wrong with the guest state.
10532 *
10533 * @param pVM The cross context VM structure.
10534 * @param pVCpu The cross context virtual CPU structure.
10535 * @param pCtx Pointer to the guest-CPU state.
10536 *
10537 * @remarks This function assumes our cache of the VMCS controls
10538 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10539 */
10540static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10541{
10542#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10543#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10544 uError = (err); \
10545 break; \
10546 } else do { } while (0)
10547
10548 int rc;
10549 uint32_t uError = VMX_IGS_ERROR;
10550 uint32_t u32Val;
10551 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10552
10553 do
10554 {
10555 /*
10556 * CR0.
10557 */
10558 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10559 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10560 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10561 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10562 if (fUnrestrictedGuest)
10563 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10564
10565 uint32_t u32GuestCR0;
10566 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10567 AssertRCBreak(rc);
10568 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10569 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10570 if ( !fUnrestrictedGuest
10571 && (u32GuestCR0 & X86_CR0_PG)
10572 && !(u32GuestCR0 & X86_CR0_PE))
10573 {
10574 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10575 }
10576
10577 /*
10578 * CR4.
10579 */
10580 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10581 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10582
10583 uint32_t u32GuestCR4;
10584 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10585 AssertRCBreak(rc);
10586 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10587 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10588
10589 /*
10590 * IA32_DEBUGCTL MSR.
10591 */
10592 uint64_t u64Val;
10593 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10594 AssertRCBreak(rc);
10595 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10596 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10597 {
10598 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10599 }
10600 uint64_t u64DebugCtlMsr = u64Val;
10601
10602#ifdef VBOX_STRICT
10603 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10604 AssertRCBreak(rc);
10605 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10606#endif
10607 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10608
10609 /*
10610 * RIP and RFLAGS.
10611 */
10612 uint32_t u32Eflags;
10613#if HC_ARCH_BITS == 64
10614 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10615 AssertRCBreak(rc);
10616 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10617 if ( !fLongModeGuest
10618 || !pCtx->cs.Attr.n.u1Long)
10619 {
10620 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10621 }
10622 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10623 * must be identical if the "IA-32e mode guest" VM-entry
10624 * control is 1 and CS.L is 1. No check applies if the
10625 * CPU supports 64 linear-address bits. */
10626
10627 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10628 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10629 AssertRCBreak(rc);
10630 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10631 VMX_IGS_RFLAGS_RESERVED);
10632 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10633 u32Eflags = u64Val;
10634#else
10635 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10636 AssertRCBreak(rc);
10637 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10638 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10639#endif
10640
10641 if ( fLongModeGuest
10642 || ( fUnrestrictedGuest
10643 && !(u32GuestCR0 & X86_CR0_PE)))
10644 {
10645 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10646 }
10647
10648 uint32_t u32EntryInfo;
10649 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10650 AssertRCBreak(rc);
10651 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10652 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10653 {
10654 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10655 }
10656
10657 /*
10658 * 64-bit checks.
10659 */
10660#if HC_ARCH_BITS == 64
10661 if (fLongModeGuest)
10662 {
10663 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10664 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10665 }
10666
10667 if ( !fLongModeGuest
10668 && (u32GuestCR4 & X86_CR4_PCIDE))
10669 {
10670 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10671 }
10672
10673 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10674 * 51:32 beyond the processor's physical-address width are 0. */
10675
10676 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10677 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10678 {
10679 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10680 }
10681
10682 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10683 AssertRCBreak(rc);
10684 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10685
10686 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10687 AssertRCBreak(rc);
10688 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10689#endif
10690
10691 /*
10692 * PERF_GLOBAL MSR.
10693 */
10694 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10695 {
10696 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10697 AssertRCBreak(rc);
10698 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10699 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10700 }
10701
10702 /*
10703 * PAT MSR.
10704 */
10705 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10706 {
10707 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10708 AssertRCBreak(rc);
10709 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10710 for (unsigned i = 0; i < 8; i++)
10711 {
10712 uint8_t u8Val = (u64Val & 0xff);
10713 if ( u8Val != 0 /* UC */
10714 && u8Val != 1 /* WC */
10715 && u8Val != 4 /* WT */
10716 && u8Val != 5 /* WP */
10717 && u8Val != 6 /* WB */
10718 && u8Val != 7 /* UC- */)
10719 {
10720 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10721 }
10722 u64Val >>= 8;
10723 }
10724 }
10725
10726 /*
10727 * EFER MSR.
10728 */
10729 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10730 {
10731 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10732 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10733 AssertRCBreak(rc);
10734 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10735 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10736 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10737 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10738 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10739 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10740 || !(u32GuestCR0 & X86_CR0_PG)
10741 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10742 VMX_IGS_EFER_LMA_LME_MISMATCH);
10743 }
10744
10745 /*
10746 * Segment registers.
10747 */
10748 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10749 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10750 if (!(u32Eflags & X86_EFL_VM))
10751 {
10752 /* CS */
10753 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10754 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10755 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10756 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10757 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10758 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10759 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10760 /* CS cannot be loaded with NULL in protected mode. */
10761 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10762 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10763 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10764 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10765 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10766 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10767 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10768 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10769 else
10770 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10771
10772 /* SS */
10773 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10774 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10775 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10776 if ( !(pCtx->cr0 & X86_CR0_PE)
10777 || pCtx->cs.Attr.n.u4Type == 3)
10778 {
10779 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10780 }
10781 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10782 {
10783 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10784 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10785 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10786 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10787 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10788 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10789 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10790 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10791 }
10792
10793 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10794 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10795 {
10796 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10797 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10798 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10799 || pCtx->ds.Attr.n.u4Type > 11
10800 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10801 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10802 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10803 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10804 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10805 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10806 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10807 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10808 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10809 }
10810 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10811 {
10812 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10813 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10814 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10815 || pCtx->es.Attr.n.u4Type > 11
10816 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10817 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10818 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10819 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10820 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10821 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10822 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10823 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10824 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10825 }
10826 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10827 {
10828 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10829 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10830 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10831 || pCtx->fs.Attr.n.u4Type > 11
10832 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10833 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10834 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10835 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10836 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10837 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10838 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10839 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10840 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10841 }
10842 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10843 {
10844 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10845 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10846 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10847 || pCtx->gs.Attr.n.u4Type > 11
10848 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10849 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10850 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10851 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10852 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10853 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10854 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10855 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10856 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10857 }
10858 /* 64-bit capable CPUs. */
10859#if HC_ARCH_BITS == 64
10860 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10861 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10862 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10863 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10864 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10865 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10866 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10867 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10868 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10869 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10870 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10871#endif
10872 }
10873 else
10874 {
10875 /* V86 mode checks. */
10876 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10877 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10878 {
10879 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10880 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10881 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10882 }
10883 else
10884 {
10885 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10886 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10887 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10888 }
10889
10890 /* CS */
10891 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10892 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10893 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10894 /* SS */
10895 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10896 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10897 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10898 /* DS */
10899 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10900 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10901 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10902 /* ES */
10903 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10904 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10905 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10906 /* FS */
10907 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10908 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10909 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10910 /* GS */
10911 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10912 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10913 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10914 /* 64-bit capable CPUs. */
10915#if HC_ARCH_BITS == 64
10916 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10917 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10918 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10919 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10920 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10921 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10922 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10923 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10924 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10925 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10926 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10927#endif
10928 }
10929
10930 /*
10931 * TR.
10932 */
10933 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10934 /* 64-bit capable CPUs. */
10935#if HC_ARCH_BITS == 64
10936 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10937#endif
10938 if (fLongModeGuest)
10939 {
10940 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10941 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10942 }
10943 else
10944 {
10945 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10946 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10947 VMX_IGS_TR_ATTR_TYPE_INVALID);
10948 }
10949 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10950 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10951 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10952 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10953 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10954 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10955 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10956 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10957
10958 /*
10959 * GDTR and IDTR.
10960 */
10961#if HC_ARCH_BITS == 64
10962 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10963 AssertRCBreak(rc);
10964 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10965
10966 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10967 AssertRCBreak(rc);
10968 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10969#endif
10970
10971 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10972 AssertRCBreak(rc);
10973 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10974
10975 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10976 AssertRCBreak(rc);
10977 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10978
10979 /*
10980 * Guest Non-Register State.
10981 */
10982 /* Activity State. */
10983 uint32_t u32ActivityState;
10984 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10985 AssertRCBreak(rc);
10986 HMVMX_CHECK_BREAK( !u32ActivityState
10987 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10988 VMX_IGS_ACTIVITY_STATE_INVALID);
10989 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10990 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10991 uint32_t u32IntrState;
10992 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10993 AssertRCBreak(rc);
10994 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10995 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10996 {
10997 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10998 }
10999
11000 /** @todo Activity state and injecting interrupts. Left as a todo since we
11001 * currently don't use activity states but ACTIVE. */
11002
11003 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11004 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11005
11006 /* Guest interruptibility-state. */
11007 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11008 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11009 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11010 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11011 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11012 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11013 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11014 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11015 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11016 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11017 {
11018 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11019 {
11020 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11021 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11022 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11023 }
11024 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11025 {
11026 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11027 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11028 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11029 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11030 }
11031 }
11032 /** @todo Assumes the processor is not in SMM. */
11033 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11034 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11035 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11036 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11037 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11038 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11039 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11040 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11041 {
11042 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11043 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11044 }
11045
11046 /* Pending debug exceptions. */
11047#if HC_ARCH_BITS == 64
11048 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11049 AssertRCBreak(rc);
11050 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11051 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11052 u32Val = u64Val; /* For pending debug exceptions checks below. */
11053#else
11054 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11055 AssertRCBreak(rc);
11056 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11057 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11058#endif
11059
11060 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11061 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11062 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11063 {
11064 if ( (u32Eflags & X86_EFL_TF)
11065 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11066 {
11067 /* Bit 14 is PendingDebug.BS. */
11068 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11069 }
11070 if ( !(u32Eflags & X86_EFL_TF)
11071 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11072 {
11073 /* Bit 14 is PendingDebug.BS. */
11074 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11075 }
11076 }
11077
11078 /* VMCS link pointer. */
11079 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11080 AssertRCBreak(rc);
11081 if (u64Val != UINT64_C(0xffffffffffffffff))
11082 {
11083 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11084 /** @todo Bits beyond the processor's physical-address width MBZ. */
11085 /** @todo 32-bit located in memory referenced by value of this field (as a
11086 * physical address) must contain the processor's VMCS revision ID. */
11087 /** @todo SMM checks. */
11088 }
11089
11090 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11091 * not using Nested Paging? */
11092 if ( pVM->hm.s.fNestedPaging
11093 && !fLongModeGuest
11094 && CPUMIsGuestInPAEModeEx(pCtx))
11095 {
11096 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11097 AssertRCBreak(rc);
11098 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11099
11100 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11101 AssertRCBreak(rc);
11102 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11103
11104 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11105 AssertRCBreak(rc);
11106 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11107
11108 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11109 AssertRCBreak(rc);
11110 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11111 }
11112
11113 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11114 if (uError == VMX_IGS_ERROR)
11115 uError = VMX_IGS_REASON_NOT_FOUND;
11116 } while (0);
11117
11118 pVCpu->hm.s.u32HMError = uError;
11119 return uError;
11120
11121#undef HMVMX_ERROR_BREAK
11122#undef HMVMX_CHECK_BREAK
11123}
11124
11125/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11126/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11127/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11128
11129/** @name VM-exit handlers.
11130 * @{
11131 */
11132
11133/**
11134 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11135 */
11136HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11137{
11138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11140 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11141 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11142 return VINF_SUCCESS;
11143 return VINF_EM_RAW_INTERRUPT;
11144}
11145
11146
11147/**
11148 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11149 */
11150HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11151{
11152 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11153 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11154
11155 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11156 AssertRCReturn(rc, rc);
11157
11158 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11159 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11160 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11161 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11162
11163 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11164 {
11165 /*
11166 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11167 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11168 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11169 *
11170 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11171 */
11172 VMXDispatchHostNmi();
11173 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11174 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11175 return VINF_SUCCESS;
11176 }
11177
11178 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11179 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11180 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11181 { /* likely */ }
11182 else
11183 {
11184 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11185 rcStrictRc1 = VINF_SUCCESS;
11186 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11187 return rcStrictRc1;
11188 }
11189
11190 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11191 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11192 switch (uIntType)
11193 {
11194 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11195 Assert(uVector == X86_XCPT_DB);
11196 /* no break */
11197 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11198 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11199 /* no break */
11200 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11201 {
11202 /*
11203 * If there's any exception caused as a result of event injection, go back to
11204 * the interpreter. The page-fault case is complicated and we manually handle
11205 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11206 * handled in hmR0VmxCheckExitDueToEventDelivery.
11207 */
11208 if (!pVCpu->hm.s.Event.fPending)
11209 { /* likely */ }
11210 else if ( uVector != X86_XCPT_PF
11211 && uVector != X86_XCPT_AC)
11212 {
11213 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11214 rc = VERR_EM_INTERPRETER;
11215 break;
11216 }
11217
11218 switch (uVector)
11219 {
11220 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11221 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11222 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11223 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11224 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11225 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11226 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11227
11228 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11229 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11230 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11231 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11232 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11233 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11234 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11235 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11236 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11237 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11238 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11239 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11240 default:
11241 {
11242 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11243 AssertRCReturn(rc, rc);
11244
11245 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11246 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11247 {
11248 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11249 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11250 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11251
11252 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11253 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11254 AssertRCReturn(rc, rc);
11255 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11256 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11257 0 /* GCPtrFaultAddress */);
11258 AssertRCReturn(rc, rc);
11259 }
11260 else
11261 {
11262 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11263 pVCpu->hm.s.u32HMError = uVector;
11264 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11265 }
11266 break;
11267 }
11268 }
11269 break;
11270 }
11271
11272 default:
11273 {
11274 pVCpu->hm.s.u32HMError = uExitIntInfo;
11275 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11276 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11277 break;
11278 }
11279 }
11280 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11281 return rc;
11282}
11283
11284
11285/**
11286 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11287 */
11288HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11289{
11290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11291
11292 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11293 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11294
11295 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11297 return VINF_SUCCESS;
11298}
11299
11300
11301/**
11302 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11303 */
11304HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11305{
11306 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11307 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11308 {
11309 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11310 HMVMX_RETURN_UNEXPECTED_EXIT();
11311 }
11312
11313 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11314
11315 /*
11316 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11317 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11318 */
11319 uint32_t uIntrState = 0;
11320 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11321 AssertRCReturn(rc, rc);
11322
11323 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11324 if ( fBlockSti
11325 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11326 {
11327 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11328 }
11329
11330 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11331 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11332
11333 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11334 return VINF_SUCCESS;
11335}
11336
11337
11338/**
11339 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11340 */
11341HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11342{
11343 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11344 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11345 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11346}
11347
11348
11349/**
11350 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11351 */
11352HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11353{
11354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11356 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11357}
11358
11359
11360/**
11361 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11362 */
11363HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11364{
11365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11366 PVM pVM = pVCpu->CTX_SUFF(pVM);
11367 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11368 if (RT_LIKELY(rc == VINF_SUCCESS))
11369 {
11370 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11371 Assert(pVmxTransient->cbInstr == 2);
11372 }
11373 else
11374 {
11375 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11376 rc = VERR_EM_INTERPRETER;
11377 }
11378 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11379 return rc;
11380}
11381
11382
11383/**
11384 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11385 */
11386HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11387{
11388 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11389 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11390 AssertRCReturn(rc, rc);
11391
11392 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11393 return VINF_EM_RAW_EMULATE_INSTR;
11394
11395 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11396 HMVMX_RETURN_UNEXPECTED_EXIT();
11397}
11398
11399
11400/**
11401 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11402 */
11403HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11404{
11405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11406 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11407 AssertRCReturn(rc, rc);
11408
11409 PVM pVM = pVCpu->CTX_SUFF(pVM);
11410 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11411 if (RT_LIKELY(rc == VINF_SUCCESS))
11412 {
11413 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11414 Assert(pVmxTransient->cbInstr == 2);
11415 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11416 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11417 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11418 }
11419 else
11420 rc = VERR_EM_INTERPRETER;
11421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11422 return rc;
11423}
11424
11425
11426/**
11427 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11428 */
11429HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11430{
11431 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11432 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11433 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11434 AssertRCReturn(rc, rc);
11435
11436 PVM pVM = pVCpu->CTX_SUFF(pVM);
11437 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11438 if (RT_SUCCESS(rc))
11439 {
11440 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11441 Assert(pVmxTransient->cbInstr == 3);
11442 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11443 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11444 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11445 }
11446 else
11447 {
11448 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11449 rc = VERR_EM_INTERPRETER;
11450 }
11451 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11452 return rc;
11453}
11454
11455
11456/**
11457 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11458 */
11459HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11460{
11461 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11462 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11463 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11464 AssertRCReturn(rc, rc);
11465
11466 PVM pVM = pVCpu->CTX_SUFF(pVM);
11467 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11468 if (RT_LIKELY(rc == VINF_SUCCESS))
11469 {
11470 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11471 Assert(pVmxTransient->cbInstr == 2);
11472 }
11473 else
11474 {
11475 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11476 rc = VERR_EM_INTERPRETER;
11477 }
11478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11479 return rc;
11480}
11481
11482
11483/**
11484 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11485 */
11486HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11487{
11488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11490
11491 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11492 if (pVCpu->hm.s.fHypercallsEnabled)
11493 {
11494#if 0
11495 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11496#else
11497 /* Aggressive state sync. for now. */
11498 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11499 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11500 AssertRCReturn(rc, rc);
11501#endif
11502
11503 /* Perform the hypercall. */
11504 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11505 if (rcStrict == VINF_SUCCESS)
11506 {
11507 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11508 AssertRCReturn(rc, rc);
11509 }
11510 else
11511 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11512 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11513 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11514
11515 /* If the hypercall changes anything other than guest's general-purpose registers,
11516 we would need to reload the guest changed bits here before VM-entry. */
11517 }
11518 else
11519 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11520
11521 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11522 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11523 {
11524 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11525 rcStrict = VINF_SUCCESS;
11526 }
11527
11528 return rcStrict;
11529}
11530
11531
11532/**
11533 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11534 */
11535HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11536{
11537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11538 PVM pVM = pVCpu->CTX_SUFF(pVM);
11539 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11540
11541 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11542 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11543 AssertRCReturn(rc, rc);
11544
11545 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11546 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11547 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11548 else
11549 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11550 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11551 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11552 return rcStrict;
11553}
11554
11555
11556/**
11557 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11558 */
11559HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11560{
11561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11562 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11563 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11564 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11565 AssertRCReturn(rc, rc);
11566
11567 PVM pVM = pVCpu->CTX_SUFF(pVM);
11568 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11569 if (RT_LIKELY(rc == VINF_SUCCESS))
11570 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11571 else
11572 {
11573 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11574 rc = VERR_EM_INTERPRETER;
11575 }
11576 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11577 return rc;
11578}
11579
11580
11581/**
11582 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11583 */
11584HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11585{
11586 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11587 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11588 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11589 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11590 AssertRCReturn(rc, rc);
11591
11592 PVM pVM = pVCpu->CTX_SUFF(pVM);
11593 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11594 rc = VBOXSTRICTRC_VAL(rc2);
11595 if (RT_LIKELY( rc == VINF_SUCCESS
11596 || rc == VINF_EM_HALT))
11597 {
11598 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11599 AssertRCReturn(rc3, rc3);
11600
11601 if ( rc == VINF_EM_HALT
11602 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11603 {
11604 rc = VINF_SUCCESS;
11605 }
11606 }
11607 else
11608 {
11609 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11610 rc = VERR_EM_INTERPRETER;
11611 }
11612 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11613 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11614 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11615 return rc;
11616}
11617
11618
11619/**
11620 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11621 */
11622HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11623{
11624 /*
11625 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11626 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11627 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11628 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11629 */
11630 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11631 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11632 HMVMX_RETURN_UNEXPECTED_EXIT();
11633}
11634
11635
11636/**
11637 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11638 */
11639HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11640{
11641 /*
11642 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11643 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11644 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11645 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11646 */
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11648 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11649 HMVMX_RETURN_UNEXPECTED_EXIT();
11650}
11651
11652
11653/**
11654 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11655 */
11656HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657{
11658 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11659 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11660 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11661 HMVMX_RETURN_UNEXPECTED_EXIT();
11662}
11663
11664
11665/**
11666 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11667 */
11668HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11669{
11670 /*
11671 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11672 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11673 * See Intel spec. 25.3 "Other Causes of VM-exits".
11674 */
11675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11676 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11677 HMVMX_RETURN_UNEXPECTED_EXIT();
11678}
11679
11680
11681/**
11682 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11683 * VM-exit.
11684 */
11685HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11686{
11687 /*
11688 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11689 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11690 *
11691 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11692 * See Intel spec. "23.8 Restrictions on VMX operation".
11693 */
11694 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11695 return VINF_SUCCESS;
11696}
11697
11698
11699/**
11700 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11701 * VM-exit.
11702 */
11703HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11704{
11705 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11706 return VINF_EM_RESET;
11707}
11708
11709
11710/**
11711 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11712 */
11713HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11714{
11715 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11716 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11717
11718 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11719 AssertRCReturn(rc, rc);
11720
11721 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11722 rc = VINF_SUCCESS;
11723 else
11724 rc = VINF_EM_HALT;
11725
11726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11727 if (rc != VINF_SUCCESS)
11728 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11729 return rc;
11730}
11731
11732
11733/**
11734 * VM-exit handler for instructions that result in a \#UD exception delivered to
11735 * the guest.
11736 */
11737HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11738{
11739 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11740 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11741 return VINF_SUCCESS;
11742}
11743
11744
11745/**
11746 * VM-exit handler for expiry of the VMX preemption timer.
11747 */
11748HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11749{
11750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11751
11752 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11753 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11754
11755 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11756 PVM pVM = pVCpu->CTX_SUFF(pVM);
11757 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11758 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11759 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11760}
11761
11762
11763/**
11764 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11765 */
11766HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11767{
11768 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11769
11770 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11771 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11772 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11773 AssertRCReturn(rc, rc);
11774
11775 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11776 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11777
11778 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11779
11780 return rcStrict;
11781}
11782
11783
11784/**
11785 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11786 */
11787HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11788{
11789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11790
11791 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11792 /** @todo implement EMInterpretInvpcid() */
11793 return VERR_EM_INTERPRETER;
11794}
11795
11796
11797/**
11798 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11799 * Error VM-exit.
11800 */
11801HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11802{
11803 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11804 AssertRCReturn(rc, rc);
11805
11806 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11807 AssertRCReturn(rc, rc);
11808
11809 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11810 NOREF(uInvalidReason);
11811
11812#ifdef VBOX_STRICT
11813 uint32_t uIntrState;
11814 RTHCUINTREG uHCReg;
11815 uint64_t u64Val;
11816 uint32_t u32Val;
11817
11818 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11819 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11820 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11821 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11822 AssertRCReturn(rc, rc);
11823
11824 Log4(("uInvalidReason %u\n", uInvalidReason));
11825 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11826 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11827 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11828 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11829
11830 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11831 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11832 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11833 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11834 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11835 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11836 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11837 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11838 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11839 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11840 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11841 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11842#else
11843 NOREF(pVmxTransient);
11844#endif
11845
11846 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11847 return VERR_VMX_INVALID_GUEST_STATE;
11848}
11849
11850
11851/**
11852 * VM-exit handler for VM-entry failure due to an MSR-load
11853 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11854 */
11855HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11856{
11857 NOREF(pVmxTransient);
11858 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11859 HMVMX_RETURN_UNEXPECTED_EXIT();
11860}
11861
11862
11863/**
11864 * VM-exit handler for VM-entry failure due to a machine-check event
11865 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11866 */
11867HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11868{
11869 NOREF(pVmxTransient);
11870 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11871 HMVMX_RETURN_UNEXPECTED_EXIT();
11872}
11873
11874
11875/**
11876 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11877 * theory.
11878 */
11879HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11880{
11881 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11882 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11883 return VERR_VMX_UNDEFINED_EXIT_CODE;
11884}
11885
11886
11887/**
11888 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11889 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11890 * Conditional VM-exit.
11891 */
11892HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11893{
11894 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11895
11896 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11897 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11898 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11899 return VERR_EM_INTERPRETER;
11900 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11901 HMVMX_RETURN_UNEXPECTED_EXIT();
11902}
11903
11904
11905/**
11906 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11907 */
11908HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11909{
11910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11911
11912 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11913 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11914 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11915 return VERR_EM_INTERPRETER;
11916 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11917 HMVMX_RETURN_UNEXPECTED_EXIT();
11918}
11919
11920
11921/**
11922 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11923 */
11924HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11925{
11926 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11927
11928 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11929 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11930 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11931 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11932 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11933 {
11934 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11935 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11936 }
11937 AssertRCReturn(rc, rc);
11938 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11939
11940#ifdef VBOX_STRICT
11941 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11942 {
11943 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11944 && pMixedCtx->ecx != MSR_K6_EFER)
11945 {
11946 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11947 pMixedCtx->ecx));
11948 HMVMX_RETURN_UNEXPECTED_EXIT();
11949 }
11950 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11951 {
11952 VMXMSREXITREAD enmRead;
11953 VMXMSREXITWRITE enmWrite;
11954 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11955 AssertRCReturn(rc2, rc2);
11956 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11957 {
11958 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11959 HMVMX_RETURN_UNEXPECTED_EXIT();
11960 }
11961 }
11962 }
11963#endif
11964
11965 PVM pVM = pVCpu->CTX_SUFF(pVM);
11966 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11967 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11968 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11970 if (RT_SUCCESS(rc))
11971 {
11972 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11973 Assert(pVmxTransient->cbInstr == 2);
11974 }
11975 return rc;
11976}
11977
11978
11979/**
11980 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11981 */
11982HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11983{
11984 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11985 PVM pVM = pVCpu->CTX_SUFF(pVM);
11986 int rc = VINF_SUCCESS;
11987
11988 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11989 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11990 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11991 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11992 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11993 {
11994 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11995 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11996 }
11997 AssertRCReturn(rc, rc);
11998 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11999
12000 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12001 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12003
12004 if (RT_SUCCESS(rc))
12005 {
12006 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12007
12008 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12009 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12010 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
12011 {
12012 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12013 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12014 EMInterpretWrmsr() changes it. */
12015 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12016 }
12017 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12018 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12019 else if (pMixedCtx->ecx == MSR_K6_EFER)
12020 {
12021 /*
12022 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12023 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12024 * the other bits as well, SCE and NXE. See @bugref{7368}.
12025 */
12026 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12027 }
12028
12029 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12030 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12031 {
12032 switch (pMixedCtx->ecx)
12033 {
12034 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12035 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12036 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12037 case MSR_K8_FS_BASE: /* no break */
12038 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12039 case MSR_K6_EFER: /* already handled above */ break;
12040 default:
12041 {
12042 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12043 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12044 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12045 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12046 break;
12047 }
12048 }
12049 }
12050#ifdef VBOX_STRICT
12051 else
12052 {
12053 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12054 switch (pMixedCtx->ecx)
12055 {
12056 case MSR_IA32_SYSENTER_CS:
12057 case MSR_IA32_SYSENTER_EIP:
12058 case MSR_IA32_SYSENTER_ESP:
12059 case MSR_K8_FS_BASE:
12060 case MSR_K8_GS_BASE:
12061 {
12062 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12063 HMVMX_RETURN_UNEXPECTED_EXIT();
12064 }
12065
12066 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12067 default:
12068 {
12069 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12070 {
12071 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12072 if (pMixedCtx->ecx != MSR_K6_EFER)
12073 {
12074 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12075 pMixedCtx->ecx));
12076 HMVMX_RETURN_UNEXPECTED_EXIT();
12077 }
12078 }
12079
12080 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12081 {
12082 VMXMSREXITREAD enmRead;
12083 VMXMSREXITWRITE enmWrite;
12084 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12085 AssertRCReturn(rc2, rc2);
12086 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12087 {
12088 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12089 HMVMX_RETURN_UNEXPECTED_EXIT();
12090 }
12091 }
12092 break;
12093 }
12094 }
12095 }
12096#endif /* VBOX_STRICT */
12097 }
12098 return rc;
12099}
12100
12101
12102/**
12103 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12104 */
12105HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12106{
12107 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12108
12109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12110 return VINF_EM_RAW_INTERRUPT;
12111}
12112
12113
12114/**
12115 * VM-exit handler for when the TPR value is lowered below the specified
12116 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12117 */
12118HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12119{
12120 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12121 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12122
12123 /*
12124 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12125 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12126 * resume guest execution.
12127 */
12128 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12130 return VINF_SUCCESS;
12131}
12132
12133
12134/**
12135 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12136 * VM-exit.
12137 *
12138 * @retval VINF_SUCCESS when guest execution can continue.
12139 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12140 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12141 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12142 * interpreter.
12143 */
12144HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12145{
12146 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12147 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12148 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12149 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12150 AssertRCReturn(rc, rc);
12151
12152 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12153 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12154 PVM pVM = pVCpu->CTX_SUFF(pVM);
12155 VBOXSTRICTRC rcStrict;
12156 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12157 switch (uAccessType)
12158 {
12159 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12160 {
12161 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12162 AssertRCReturn(rc, rc);
12163
12164 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12165 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12166 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12167 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12168 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12169 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12170 {
12171 case 0: /* CR0 */
12172 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12173 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12174 break;
12175 case 2: /* CR2 */
12176 /* Nothing to do here, CR2 it's not part of the VMCS. */
12177 break;
12178 case 3: /* CR3 */
12179 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12180 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12181 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12182 break;
12183 case 4: /* CR4 */
12184 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12185 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12186 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12187 break;
12188 case 8: /* CR8 */
12189 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12190 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12191 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12192 break;
12193 default:
12194 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12195 break;
12196 }
12197
12198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12199 break;
12200 }
12201
12202 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12203 {
12204 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12205 AssertRCReturn(rc, rc);
12206
12207 Assert( !pVM->hm.s.fNestedPaging
12208 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12209 || pVCpu->hm.s.fUsingDebugLoop
12210 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12211
12212 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12213 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12214 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12215
12216 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12217 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12218 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12219 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12220 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12221 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12222 VBOXSTRICTRC_VAL(rcStrict)));
12223 break;
12224 }
12225
12226 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12227 {
12228 AssertRCReturn(rc, rc);
12229 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12230 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12231 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12232 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12233 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12234 break;
12235 }
12236
12237 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12238 {
12239 AssertRCReturn(rc, rc);
12240 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12241 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12242 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12243 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12244 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12245 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12246 break;
12247 }
12248
12249 default:
12250 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12251 VERR_VMX_UNEXPECTED_EXCEPTION);
12252 }
12253
12254 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12255 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12256 NOREF(pVM);
12257 return rcStrict;
12258}
12259
12260
12261/**
12262 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12263 * VM-exit.
12264 */
12265HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12266{
12267 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12268 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12269
12270 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12271 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12272 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12273 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12274 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12275 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12276 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12277 AssertRCReturn(rc2, rc2);
12278
12279 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12280 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12281 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12282 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12283 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12284 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12285 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12286 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12287 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12288
12289 /* I/O operation lookup arrays. */
12290 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12291 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12292
12293 VBOXSTRICTRC rcStrict;
12294 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12295 uint32_t const cbInstr = pVmxTransient->cbInstr;
12296 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12297 PVM pVM = pVCpu->CTX_SUFF(pVM);
12298 if (fIOString)
12299 {
12300#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12301 See @bugref{5752#c158}. Should work now. */
12302 /*
12303 * INS/OUTS - I/O String instruction.
12304 *
12305 * Use instruction-information if available, otherwise fall back on
12306 * interpreting the instruction.
12307 */
12308 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12309 fIOWrite ? 'w' : 'r'));
12310 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12311 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12312 {
12313 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12314 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12315 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12316 AssertRCReturn(rc2, rc2);
12317 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12318 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12319 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12320 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12321 if (fIOWrite)
12322 {
12323 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12324 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12325 }
12326 else
12327 {
12328 /*
12329 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12330 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12331 * See Intel Instruction spec. for "INS".
12332 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12333 */
12334 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12335 }
12336 }
12337 else
12338 {
12339 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12340 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12341 AssertRCReturn(rc2, rc2);
12342 rcStrict = IEMExecOne(pVCpu);
12343 }
12344 /** @todo IEM needs to be setting these flags somehow. */
12345 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12346 fUpdateRipAlready = true;
12347#else
12348 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12349 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12350 if (RT_SUCCESS(rcStrict))
12351 {
12352 if (fIOWrite)
12353 {
12354 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12355 (DISCPUMODE)pDis->uAddrMode, cbValue);
12356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12357 }
12358 else
12359 {
12360 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12361 (DISCPUMODE)pDis->uAddrMode, cbValue);
12362 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12363 }
12364 }
12365 else
12366 {
12367 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12368 pMixedCtx->rip));
12369 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12370 }
12371#endif
12372 }
12373 else
12374 {
12375 /*
12376 * IN/OUT - I/O instruction.
12377 */
12378 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12379 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12380 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12381 if (fIOWrite)
12382 {
12383 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12384 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12385 }
12386 else
12387 {
12388 uint32_t u32Result = 0;
12389 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12390 if (IOM_SUCCESS(rcStrict))
12391 {
12392 /* Save result of I/O IN instr. in AL/AX/EAX. */
12393 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12394 }
12395 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12396 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12397 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12398 }
12399 }
12400
12401 if (IOM_SUCCESS(rcStrict))
12402 {
12403 if (!fUpdateRipAlready)
12404 {
12405 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12406 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12407 }
12408
12409 /*
12410 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12411 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12412 */
12413 if (fIOString)
12414 {
12415 /** @todo Single-step for INS/OUTS with REP prefix? */
12416 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12417 }
12418 else if ( !fDbgStepping
12419 && fGstStepping)
12420 {
12421 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12422 }
12423
12424 /*
12425 * If any I/O breakpoints are armed, we need to check if one triggered
12426 * and take appropriate action.
12427 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12428 */
12429 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12430 AssertRCReturn(rc2, rc2);
12431
12432 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12433 * execution engines about whether hyper BPs and such are pending. */
12434 uint32_t const uDr7 = pMixedCtx->dr[7];
12435 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12436 && X86_DR7_ANY_RW_IO(uDr7)
12437 && (pMixedCtx->cr4 & X86_CR4_DE))
12438 || DBGFBpIsHwIoArmed(pVM)))
12439 {
12440 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12441
12442 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12443 VMMRZCallRing3Disable(pVCpu);
12444 HM_DISABLE_PREEMPT();
12445
12446 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12447
12448 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12449 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12450 {
12451 /* Raise #DB. */
12452 if (fIsGuestDbgActive)
12453 ASMSetDR6(pMixedCtx->dr[6]);
12454 if (pMixedCtx->dr[7] != uDr7)
12455 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12456
12457 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12458 }
12459 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12460 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12461 else if ( rcStrict2 != VINF_SUCCESS
12462 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12463 rcStrict = rcStrict2;
12464 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12465
12466 HM_RESTORE_PREEMPT();
12467 VMMRZCallRing3Enable(pVCpu);
12468 }
12469 }
12470
12471#ifdef VBOX_STRICT
12472 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12473 Assert(!fIOWrite);
12474 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12475 Assert(fIOWrite);
12476 else
12477 {
12478#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12479 * statuses, that the VMM device and some others may return. See
12480 * IOM_SUCCESS() for guidance. */
12481 AssertMsg( RT_FAILURE(rcStrict)
12482 || rcStrict == VINF_SUCCESS
12483 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12484 || rcStrict == VINF_EM_DBG_BREAKPOINT
12485 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12486 || rcStrict == VINF_EM_RAW_TO_R3
12487 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12488#endif
12489 }
12490#endif
12491
12492 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12493 return rcStrict;
12494}
12495
12496
12497/**
12498 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12499 * VM-exit.
12500 */
12501HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12502{
12503 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12504
12505 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12506 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12507 AssertRCReturn(rc, rc);
12508 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12509 {
12510 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12511 AssertRCReturn(rc, rc);
12512 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12513 {
12514 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12515
12516 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12517 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12518
12519 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12520 Assert(!pVCpu->hm.s.Event.fPending);
12521 pVCpu->hm.s.Event.fPending = true;
12522 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12523 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12524 AssertRCReturn(rc, rc);
12525 if (fErrorCodeValid)
12526 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12527 else
12528 pVCpu->hm.s.Event.u32ErrCode = 0;
12529 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12530 && uVector == X86_XCPT_PF)
12531 {
12532 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12533 }
12534
12535 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12537 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12538 }
12539 }
12540
12541 /* Fall back to the interpreter to emulate the task-switch. */
12542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12543 return VERR_EM_INTERPRETER;
12544}
12545
12546
12547/**
12548 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12549 */
12550HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12551{
12552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12553 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12554 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12555 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12556 AssertRCReturn(rc, rc);
12557 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12558 return VINF_EM_DBG_STEPPED;
12559}
12560
12561
12562/**
12563 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12564 */
12565HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12566{
12567 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12568
12569 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12570
12571 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12572 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12573 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12574 {
12575 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12576 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12577 {
12578 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12579 return VERR_EM_INTERPRETER;
12580 }
12581 }
12582 else
12583 {
12584 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12585 rcStrict1 = VINF_SUCCESS;
12586 return rcStrict1;
12587 }
12588
12589#if 0
12590 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12591 * just sync the whole thing. */
12592 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12593#else
12594 /* Aggressive state sync. for now. */
12595 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12596 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12597 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12598#endif
12599 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12600 AssertRCReturn(rc, rc);
12601
12602 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12603 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12604 VBOXSTRICTRC rcStrict2;
12605 switch (uAccessType)
12606 {
12607 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12608 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12609 {
12610 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12611 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12612 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12613
12614 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12615 GCPhys &= PAGE_BASE_GC_MASK;
12616 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12617 PVM pVM = pVCpu->CTX_SUFF(pVM);
12618 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12619 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12620
12621 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12622 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12623 CPUMCTX2CORE(pMixedCtx), GCPhys);
12624 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12625 if ( rcStrict2 == VINF_SUCCESS
12626 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12627 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12628 {
12629 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12630 | HM_CHANGED_GUEST_RSP
12631 | HM_CHANGED_GUEST_RFLAGS
12632 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12633 rcStrict2 = VINF_SUCCESS;
12634 }
12635 break;
12636 }
12637
12638 default:
12639 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12640 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12641 break;
12642 }
12643
12644 if (rcStrict2 != VINF_SUCCESS)
12645 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12646 return rcStrict2;
12647}
12648
12649
12650/**
12651 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12652 * VM-exit.
12653 */
12654HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12655{
12656 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12657
12658 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12659 if (pVmxTransient->fWasGuestDebugStateActive)
12660 {
12661 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12662 HMVMX_RETURN_UNEXPECTED_EXIT();
12663 }
12664
12665 if ( !pVCpu->hm.s.fSingleInstruction
12666 && !pVmxTransient->fWasHyperDebugStateActive)
12667 {
12668 Assert(!DBGFIsStepping(pVCpu));
12669 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12670
12671 /* Don't intercept MOV DRx any more. */
12672 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12673 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12674 AssertRCReturn(rc, rc);
12675
12676 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12677 VMMRZCallRing3Disable(pVCpu);
12678 HM_DISABLE_PREEMPT();
12679
12680 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12681 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12682 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12683
12684 HM_RESTORE_PREEMPT();
12685 VMMRZCallRing3Enable(pVCpu);
12686
12687#ifdef VBOX_WITH_STATISTICS
12688 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12689 AssertRCReturn(rc, rc);
12690 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12691 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12692 else
12693 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12694#endif
12695 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12696 return VINF_SUCCESS;
12697 }
12698
12699 /*
12700 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12701 * Update the segment registers and DR7 from the CPU.
12702 */
12703 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12704 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12705 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12706 AssertRCReturn(rc, rc);
12707 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12708
12709 PVM pVM = pVCpu->CTX_SUFF(pVM);
12710 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12711 {
12712 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12713 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12714 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12715 if (RT_SUCCESS(rc))
12716 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12718 }
12719 else
12720 {
12721 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12722 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12723 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12725 }
12726
12727 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12728 if (RT_SUCCESS(rc))
12729 {
12730 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12731 AssertRCReturn(rc2, rc2);
12732 return VINF_SUCCESS;
12733 }
12734 return rc;
12735}
12736
12737
12738/**
12739 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12740 * Conditional VM-exit.
12741 */
12742HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12743{
12744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12745 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12746
12747 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12748 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12749 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12750 {
12751 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12752 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12753 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12754 {
12755 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12756 return VERR_EM_INTERPRETER;
12757 }
12758 }
12759 else
12760 {
12761 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12762 rcStrict1 = VINF_SUCCESS;
12763 return rcStrict1;
12764 }
12765
12766 RTGCPHYS GCPhys = 0;
12767 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12768
12769#if 0
12770 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12771#else
12772 /* Aggressive state sync. for now. */
12773 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12774 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12775 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12776#endif
12777 AssertRCReturn(rc, rc);
12778
12779 /*
12780 * If we succeed, resume guest execution.
12781 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12782 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12783 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12784 * weird case. See @bugref{6043}.
12785 */
12786 PVM pVM = pVCpu->CTX_SUFF(pVM);
12787 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12788 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12789 if ( rcStrict2 == VINF_SUCCESS
12790 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12791 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12792 {
12793 /* Successfully handled MMIO operation. */
12794 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12795 | HM_CHANGED_GUEST_RSP
12796 | HM_CHANGED_GUEST_RFLAGS
12797 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12798 return VINF_SUCCESS;
12799 }
12800 return rcStrict2;
12801}
12802
12803
12804/**
12805 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12806 * VM-exit.
12807 */
12808HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12809{
12810 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12811 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12812
12813 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12814 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12815 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12816 {
12817 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12818 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12819 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12820 }
12821 else
12822 {
12823 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12824 rcStrict1 = VINF_SUCCESS;
12825 return rcStrict1;
12826 }
12827
12828 RTGCPHYS GCPhys = 0;
12829 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12830 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12831#if 0
12832 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12833#else
12834 /* Aggressive state sync. for now. */
12835 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12836 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12837 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12838#endif
12839 AssertRCReturn(rc, rc);
12840
12841 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12842 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12843
12844 RTGCUINT uErrorCode = 0;
12845 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12846 uErrorCode |= X86_TRAP_PF_ID;
12847 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12848 uErrorCode |= X86_TRAP_PF_RW;
12849 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12850 uErrorCode |= X86_TRAP_PF_P;
12851
12852 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12853
12854 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12855 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12856
12857 /* Handle the pagefault trap for the nested shadow table. */
12858 PVM pVM = pVCpu->CTX_SUFF(pVM);
12859 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12860 TRPMResetTrap(pVCpu);
12861
12862 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12863 if ( rcStrict2 == VINF_SUCCESS
12864 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12865 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12866 {
12867 /* Successfully synced our nested page tables. */
12868 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12869 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12870 | HM_CHANGED_GUEST_RSP
12871 | HM_CHANGED_GUEST_RFLAGS);
12872 return VINF_SUCCESS;
12873 }
12874
12875 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12876 return rcStrict2;
12877}
12878
12879/** @} */
12880
12881/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12882/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12883/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12884
12885/** @name VM-exit exception handlers.
12886 * @{
12887 */
12888
12889/**
12890 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12891 */
12892static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12893{
12894 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12896
12897 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12898 AssertRCReturn(rc, rc);
12899
12900 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12901 {
12902 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12903 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12904
12905 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12906 * provides VM-exit instruction length. If this causes problem later,
12907 * disassemble the instruction like it's done on AMD-V. */
12908 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12909 AssertRCReturn(rc2, rc2);
12910 return rc;
12911 }
12912
12913 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12914 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12915 return rc;
12916}
12917
12918
12919/**
12920 * VM-exit exception handler for \#BP (Breakpoint exception).
12921 */
12922static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12923{
12924 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12926
12927 /** @todo Try optimize this by not saving the entire guest state unless
12928 * really needed. */
12929 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12930 AssertRCReturn(rc, rc);
12931
12932 PVM pVM = pVCpu->CTX_SUFF(pVM);
12933 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12934 if (rc == VINF_EM_RAW_GUEST_TRAP)
12935 {
12936 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12937 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12938 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12939 AssertRCReturn(rc, rc);
12940
12941 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12942 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12943 }
12944
12945 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12946 return rc;
12947}
12948
12949
12950/**
12951 * VM-exit exception handler for \#AC (alignment check exception).
12952 */
12953static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12954{
12955 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12956
12957 /*
12958 * Re-inject it. We'll detect any nesting before getting here.
12959 */
12960 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12961 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12962 AssertRCReturn(rc, rc);
12963 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12964
12965 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12966 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12967 return VINF_SUCCESS;
12968}
12969
12970
12971/**
12972 * VM-exit exception handler for \#DB (Debug exception).
12973 */
12974static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12975{
12976 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12977 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12978 Log6(("XcptDB\n"));
12979
12980 /*
12981 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12982 * for processing.
12983 */
12984 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12985 AssertRCReturn(rc, rc);
12986
12987 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12988 uint64_t uDR6 = X86_DR6_INIT_VAL;
12989 uDR6 |= ( pVmxTransient->uExitQualification
12990 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12991
12992 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12993 if (rc == VINF_EM_RAW_GUEST_TRAP)
12994 {
12995 /*
12996 * The exception was for the guest. Update DR6, DR7.GD and
12997 * IA32_DEBUGCTL.LBR before forwarding it.
12998 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12999 */
13000 VMMRZCallRing3Disable(pVCpu);
13001 HM_DISABLE_PREEMPT();
13002
13003 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13004 pMixedCtx->dr[6] |= uDR6;
13005 if (CPUMIsGuestDebugStateActive(pVCpu))
13006 ASMSetDR6(pMixedCtx->dr[6]);
13007
13008 HM_RESTORE_PREEMPT();
13009 VMMRZCallRing3Enable(pVCpu);
13010
13011 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13012 AssertRCReturn(rc, rc);
13013
13014 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13015 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13016
13017 /* Paranoia. */
13018 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13019 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13020
13021 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13022 AssertRCReturn(rc, rc);
13023
13024 /*
13025 * Raise #DB in the guest.
13026 *
13027 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13028 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13029 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13030 *
13031 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13032 */
13033 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13034 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13035 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13036 AssertRCReturn(rc, rc);
13037 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13038 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13039 return VINF_SUCCESS;
13040 }
13041
13042 /*
13043 * Not a guest trap, must be a hypervisor related debug event then.
13044 * Update DR6 in case someone is interested in it.
13045 */
13046 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13047 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13048 CPUMSetHyperDR6(pVCpu, uDR6);
13049
13050 return rc;
13051}
13052
13053
13054/**
13055 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13056 * point exception).
13057 */
13058static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13059{
13060 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13061
13062 /* We require CR0 and EFER. EFER is always up-to-date. */
13063 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13064 AssertRCReturn(rc, rc);
13065
13066 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13067 VMMRZCallRing3Disable(pVCpu);
13068 HM_DISABLE_PREEMPT();
13069
13070 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13071 if (pVmxTransient->fWasGuestFPUStateActive)
13072 {
13073 rc = VINF_EM_RAW_GUEST_TRAP;
13074 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13075 }
13076 else
13077 {
13078#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13079 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13080#endif
13081 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13082 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13083 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13084 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13085 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13086 }
13087
13088 HM_RESTORE_PREEMPT();
13089 VMMRZCallRing3Enable(pVCpu);
13090
13091 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13092 {
13093 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13094 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13095 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13096 pVCpu->hm.s.fPreloadGuestFpu = true;
13097 }
13098 else
13099 {
13100 /* Forward #NM to the guest. */
13101 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13102 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13103 AssertRCReturn(rc, rc);
13104 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13105 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13107 }
13108
13109 return VINF_SUCCESS;
13110}
13111
13112
13113/**
13114 * VM-exit exception handler for \#GP (General-protection exception).
13115 *
13116 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13117 */
13118static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13119{
13120 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13121 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13122
13123 int rc;
13124 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13125 { /* likely */ }
13126 else
13127 {
13128#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13129 Assert(pVCpu->hm.s.fUsingDebugLoop);
13130#endif
13131 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13132 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13133 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13134 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13135 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13136 AssertRCReturn(rc, rc);
13137 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13138 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13139 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13140 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13141 return rc;
13142 }
13143
13144 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13145 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13146
13147 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13148 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13149 AssertRCReturn(rc, rc);
13150
13151 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13152 uint32_t cbOp = 0;
13153 PVM pVM = pVCpu->CTX_SUFF(pVM);
13154 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13155 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13156 if (RT_SUCCESS(rc))
13157 {
13158 rc = VINF_SUCCESS;
13159 Assert(cbOp == pDis->cbInstr);
13160 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13161 switch (pDis->pCurInstr->uOpcode)
13162 {
13163 case OP_CLI:
13164 {
13165 pMixedCtx->eflags.Bits.u1IF = 0;
13166 pMixedCtx->eflags.Bits.u1RF = 0;
13167 pMixedCtx->rip += pDis->cbInstr;
13168 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13169 if ( !fDbgStepping
13170 && pMixedCtx->eflags.Bits.u1TF)
13171 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13172 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13173 break;
13174 }
13175
13176 case OP_STI:
13177 {
13178 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13179 pMixedCtx->eflags.Bits.u1IF = 1;
13180 pMixedCtx->eflags.Bits.u1RF = 0;
13181 pMixedCtx->rip += pDis->cbInstr;
13182 if (!fOldIF)
13183 {
13184 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13185 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13186 }
13187 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13188 if ( !fDbgStepping
13189 && pMixedCtx->eflags.Bits.u1TF)
13190 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13191 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13192 break;
13193 }
13194
13195 case OP_HLT:
13196 {
13197 rc = VINF_EM_HALT;
13198 pMixedCtx->rip += pDis->cbInstr;
13199 pMixedCtx->eflags.Bits.u1RF = 0;
13200 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13202 break;
13203 }
13204
13205 case OP_POPF:
13206 {
13207 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13208 uint32_t cbParm;
13209 uint32_t uMask;
13210 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13211 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13212 {
13213 cbParm = 4;
13214 uMask = 0xffffffff;
13215 }
13216 else
13217 {
13218 cbParm = 2;
13219 uMask = 0xffff;
13220 }
13221
13222 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13223 RTGCPTR GCPtrStack = 0;
13224 X86EFLAGS Eflags;
13225 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13226 &GCPtrStack);
13227 if (RT_SUCCESS(rc))
13228 {
13229 Assert(sizeof(Eflags.u32) >= cbParm);
13230 Eflags.u32 = 0;
13231 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13232 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13233 }
13234 if (RT_FAILURE(rc))
13235 {
13236 rc = VERR_EM_INTERPRETER;
13237 break;
13238 }
13239 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13240 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13241 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13242 pMixedCtx->esp += cbParm;
13243 pMixedCtx->esp &= uMask;
13244 pMixedCtx->rip += pDis->cbInstr;
13245 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13246 | HM_CHANGED_GUEST_RSP
13247 | HM_CHANGED_GUEST_RFLAGS);
13248 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13249 POPF restores EFLAGS.TF. */
13250 if ( !fDbgStepping
13251 && fGstStepping)
13252 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13254 break;
13255 }
13256
13257 case OP_PUSHF:
13258 {
13259 uint32_t cbParm;
13260 uint32_t uMask;
13261 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13262 {
13263 cbParm = 4;
13264 uMask = 0xffffffff;
13265 }
13266 else
13267 {
13268 cbParm = 2;
13269 uMask = 0xffff;
13270 }
13271
13272 /* Get the stack pointer & push the contents of eflags onto the stack. */
13273 RTGCPTR GCPtrStack = 0;
13274 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13275 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13276 if (RT_FAILURE(rc))
13277 {
13278 rc = VERR_EM_INTERPRETER;
13279 break;
13280 }
13281 X86EFLAGS Eflags = pMixedCtx->eflags;
13282 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13283 Eflags.Bits.u1RF = 0;
13284 Eflags.Bits.u1VM = 0;
13285
13286 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13287 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13288 {
13289 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13290 rc = VERR_EM_INTERPRETER;
13291 break;
13292 }
13293 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13294 pMixedCtx->esp -= cbParm;
13295 pMixedCtx->esp &= uMask;
13296 pMixedCtx->rip += pDis->cbInstr;
13297 pMixedCtx->eflags.Bits.u1RF = 0;
13298 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13299 | HM_CHANGED_GUEST_RSP
13300 | HM_CHANGED_GUEST_RFLAGS);
13301 if ( !fDbgStepping
13302 && pMixedCtx->eflags.Bits.u1TF)
13303 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13305 break;
13306 }
13307
13308 case OP_IRET:
13309 {
13310 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13311 * instruction reference. */
13312 RTGCPTR GCPtrStack = 0;
13313 uint32_t uMask = 0xffff;
13314 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13315 uint16_t aIretFrame[3];
13316 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13317 {
13318 rc = VERR_EM_INTERPRETER;
13319 break;
13320 }
13321 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13322 &GCPtrStack);
13323 if (RT_SUCCESS(rc))
13324 {
13325 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13326 PGMACCESSORIGIN_HM));
13327 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13328 }
13329 if (RT_FAILURE(rc))
13330 {
13331 rc = VERR_EM_INTERPRETER;
13332 break;
13333 }
13334 pMixedCtx->eip = 0;
13335 pMixedCtx->ip = aIretFrame[0];
13336 pMixedCtx->cs.Sel = aIretFrame[1];
13337 pMixedCtx->cs.ValidSel = aIretFrame[1];
13338 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13339 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13340 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13341 pMixedCtx->sp += sizeof(aIretFrame);
13342 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13343 | HM_CHANGED_GUEST_SEGMENT_REGS
13344 | HM_CHANGED_GUEST_RSP
13345 | HM_CHANGED_GUEST_RFLAGS);
13346 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13347 if ( !fDbgStepping
13348 && fGstStepping)
13349 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13350 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13352 break;
13353 }
13354
13355 case OP_INT:
13356 {
13357 uint16_t uVector = pDis->Param1.uValue & 0xff;
13358 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13359 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13361 break;
13362 }
13363
13364 case OP_INTO:
13365 {
13366 if (pMixedCtx->eflags.Bits.u1OF)
13367 {
13368 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13369 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13370 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13371 }
13372 else
13373 {
13374 pMixedCtx->eflags.Bits.u1RF = 0;
13375 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13376 }
13377 break;
13378 }
13379
13380 default:
13381 {
13382 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13383 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13384 EMCODETYPE_SUPERVISOR);
13385 rc = VBOXSTRICTRC_VAL(rc2);
13386 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13387 /** @todo We have to set pending-debug exceptions here when the guest is
13388 * single-stepping depending on the instruction that was interpreted. */
13389 Log4(("#GP rc=%Rrc\n", rc));
13390 break;
13391 }
13392 }
13393 }
13394 else
13395 rc = VERR_EM_INTERPRETER;
13396
13397 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13398 ("#GP Unexpected rc=%Rrc\n", rc));
13399 return rc;
13400}
13401
13402
13403/**
13404 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13405 * the exception reported in the VMX transient structure back into the VM.
13406 *
13407 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13408 * up-to-date.
13409 */
13410static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13411{
13412 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13413#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13414 Assert(pVCpu->hm.s.fUsingDebugLoop);
13415#endif
13416
13417 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13418 hmR0VmxCheckExitDueToEventDelivery(). */
13419 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13420 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13421 AssertRCReturn(rc, rc);
13422 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13423
13424#ifdef DEBUG_ramshankar
13425 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13426 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13427 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13428#endif
13429
13430 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13431 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13432 return VINF_SUCCESS;
13433}
13434
13435
13436/**
13437 * VM-exit exception handler for \#PF (Page-fault exception).
13438 */
13439static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13440{
13441 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13442 PVM pVM = pVCpu->CTX_SUFF(pVM);
13443 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13444 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13445 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13446 AssertRCReturn(rc, rc);
13447
13448 if (!pVM->hm.s.fNestedPaging)
13449 { /* likely */ }
13450 else
13451 {
13452#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13453 Assert(pVCpu->hm.s.fUsingDebugLoop);
13454#endif
13455 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13456 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13457 {
13458 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13459 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13460 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13461 }
13462 else
13463 {
13464 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13465 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13466 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13467 }
13468 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13469 return rc;
13470 }
13471
13472 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13473 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13474 if (pVmxTransient->fVectoringPF)
13475 {
13476 Assert(pVCpu->hm.s.Event.fPending);
13477 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13478 }
13479
13480 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13481 AssertRCReturn(rc, rc);
13482
13483 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13484 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13485
13486 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13487 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13488 (RTGCPTR)pVmxTransient->uExitQualification);
13489
13490 Log4(("#PF: rc=%Rrc\n", rc));
13491 if (rc == VINF_SUCCESS)
13492 {
13493#if 0
13494 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13495 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13496 * memory? We don't update the whole state here... */
13497 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13498 | HM_CHANGED_GUEST_RSP
13499 | HM_CHANGED_GUEST_RFLAGS
13500 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13501#else
13502 /*
13503 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13504 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13505 */
13506 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13507 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13508#endif
13509 TRPMResetTrap(pVCpu);
13510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13511 return rc;
13512 }
13513
13514 if (rc == VINF_EM_RAW_GUEST_TRAP)
13515 {
13516 if (!pVmxTransient->fVectoringDoublePF)
13517 {
13518 /* It's a guest page fault and needs to be reflected to the guest. */
13519 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13520 TRPMResetTrap(pVCpu);
13521 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13522 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13523 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13524 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13525 }
13526 else
13527 {
13528 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13529 TRPMResetTrap(pVCpu);
13530 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13531 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13532 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13533 }
13534
13535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13536 return VINF_SUCCESS;
13537 }
13538
13539 TRPMResetTrap(pVCpu);
13540 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13541 return rc;
13542}
13543
13544/** @} */
13545
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette