VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 61743

Last change on this file since 61743 was 61743, checked in by vboxsync, 9 years ago

VMM/HMVMXR0: comments, removed old todos.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 579.4 KB
Line 
1/* $Id: HMVMXR0.cpp 61743 2016-06-17 13:32:54Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#ifdef VBOX_WITH_REM
35# include <VBox/vmm/rem.h>
36#endif
37#ifdef VBOX_WITH_NEW_APIC
38# include <VBox/vmm/apic.h>
39#endif
40#include "HMInternal.h"
41#include <VBox/vmm/vm.h>
42#include "HMVMXR0.h"
43#include "dtrace/VBoxVMM.h"
44
45#ifdef DEBUG_ramshankar
46# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
47# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
49# define HMVMX_ALWAYS_CHECK_GUEST_STATE
50# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
51# define HMVMX_ALWAYS_TRAP_PF
52# define HMVMX_ALWAYS_SWAP_FPU_STATE
53# define HMVMX_ALWAYS_FLUSH_TLB
54# define HMVMX_ALWAYS_SWAP_EFER
55#endif
56
57
58/*********************************************************************************************************************************
59* Defined Constants And Macros *
60*********************************************************************************************************************************/
61/** Use the function table. */
62#define HMVMX_USE_FUNCTION_TABLE
63
64/** Determine which tagged-TLB flush handler to use. */
65#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70/** @name Updated-guest-state flags.
71 * @{ */
72#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
73#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
74#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
75#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
76#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
77#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
78#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
79#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
80#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
81#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
82#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
83#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
84#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
85#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
86#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
87#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
88#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
89#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
90#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
91#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
92#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
93 | HMVMX_UPDATED_GUEST_RSP \
94 | HMVMX_UPDATED_GUEST_RFLAGS \
95 | HMVMX_UPDATED_GUEST_CR0 \
96 | HMVMX_UPDATED_GUEST_CR3 \
97 | HMVMX_UPDATED_GUEST_CR4 \
98 | HMVMX_UPDATED_GUEST_GDTR \
99 | HMVMX_UPDATED_GUEST_IDTR \
100 | HMVMX_UPDATED_GUEST_LDTR \
101 | HMVMX_UPDATED_GUEST_TR \
102 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103 | HMVMX_UPDATED_GUEST_DEBUG \
104 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
109 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
110 | HMVMX_UPDATED_GUEST_INTR_STATE \
111 | HMVMX_UPDATED_GUEST_APIC_STATE)
112/** @} */
113
114/** @name
115 * Flags to skip redundant reads of some common VMCS fields that are not part of
116 * the guest-CPU state but are in the transient structure.
117 */
118#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually except:
143 * - \#NM, \#MF handled in hmR0VmxLoadSharedCR0().
144 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
145 * due to bugs in Intel CPUs.
146 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
147 * support.
148 */
149#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
150 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
151 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
152 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
153 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
154 /* RT_BIT(X86_XCPT_MF) always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
155 | RT_BIT(X86_XCPT_XF))
156
157/**
158 * Exception bitmap mask for all contributory exceptions.
159 *
160 * Page fault is deliberately excluded here as it's conditional as to whether
161 * it's contributory or benign. Page faults are handled separately.
162 */
163#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
164 | RT_BIT(X86_XCPT_DE))
165
166/** Maximum VM-instruction error number. */
167#define HMVMX_INSTR_ERROR_MAX 28
168
169/** Profiling macro. */
170#ifdef HM_PROFILE_EXIT_DISPATCH
171# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
173#else
174# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
175# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
176#endif
177
178/** Assert that preemption is disabled or covered by thread-context hooks. */
179#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
180 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
181
182/** Assert that we haven't migrated CPUs when thread-context hooks are not
183 * used. */
184#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
185 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
186 ("Illegal migration! Entered on CPU %u Current %u\n", \
187 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
188
189/** Helper macro for VM-exit handlers called unexpectedly. */
190#define HMVMX_RETURN_UNEXPECTED_EXIT() \
191 do { \
192 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
193 return VERR_VMX_UNEXPECTED_EXIT; \
194 } while (0)
195
196
197/*********************************************************************************************************************************
198* Structures and Typedefs *
199*********************************************************************************************************************************/
200/**
201 * VMX transient state.
202 *
203 * A state structure for holding miscellaneous information across
204 * VMX non-root operation and restored after the transition.
205 */
206typedef struct VMXTRANSIENT
207{
208 /** The host's rflags/eflags. */
209 RTCCUINTREG fEFlags;
210#if HC_ARCH_BITS == 32
211 uint32_t u32Alignment0;
212#endif
213 /** The guest's TPR value used for TPR shadowing. */
214 uint8_t u8GuestTpr;
215 /** Alignment. */
216 uint8_t abAlignment0[7];
217
218 /** The basic VM-exit reason. */
219 uint16_t uExitReason;
220 /** Alignment. */
221 uint16_t u16Alignment0;
222 /** The VM-exit interruption error code. */
223 uint32_t uExitIntErrorCode;
224 /** The VM-exit exit code qualification. */
225 uint64_t uExitQualification;
226
227 /** The VM-exit interruption-information field. */
228 uint32_t uExitIntInfo;
229 /** The VM-exit instruction-length field. */
230 uint32_t cbInstr;
231 /** The VM-exit instruction-information field. */
232 union
233 {
234 /** Plain unsigned int representation. */
235 uint32_t u;
236 /** INS and OUTS information. */
237 struct
238 {
239 uint32_t u7Reserved0 : 7;
240 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
241 uint32_t u3AddrSize : 3;
242 uint32_t u5Reserved1 : 5;
243 /** The segment register (X86_SREG_XXX). */
244 uint32_t iSegReg : 3;
245 uint32_t uReserved2 : 14;
246 } StrIo;
247 } ExitInstrInfo;
248 /** Whether the VM-entry failed or not. */
249 bool fVMEntryFailed;
250 /** Alignment. */
251 uint8_t abAlignment1[3];
252
253 /** The VM-entry interruption-information field. */
254 uint32_t uEntryIntInfo;
255 /** The VM-entry exception error code field. */
256 uint32_t uEntryXcptErrorCode;
257 /** The VM-entry instruction length field. */
258 uint32_t cbEntryInstr;
259
260 /** IDT-vectoring information field. */
261 uint32_t uIdtVectoringInfo;
262 /** IDT-vectoring error code. */
263 uint32_t uIdtVectoringErrorCode;
264
265 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
266 uint32_t fVmcsFieldsRead;
267
268 /** Whether the guest FPU was active at the time of VM-exit. */
269 bool fWasGuestFPUStateActive;
270 /** Whether the guest debug state was active at the time of VM-exit. */
271 bool fWasGuestDebugStateActive;
272 /** Whether the hyper debug state was active at the time of VM-exit. */
273 bool fWasHyperDebugStateActive;
274 /** Whether TSC-offsetting should be setup before VM-entry. */
275 bool fUpdateTscOffsettingAndPreemptTimer;
276 /** Whether the VM-exit was caused by a page-fault during delivery of a
277 * contributory exception or a page-fault. */
278 bool fVectoringDoublePF;
279 /** Whether the VM-exit was caused by a page-fault during delivery of an
280 * external interrupt or NMI. */
281 bool fVectoringPF;
282} VMXTRANSIENT;
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
286AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
287AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
288/** Pointer to VMX transient state. */
289typedef VMXTRANSIENT *PVMXTRANSIENT;
290
291
292/**
293 * MSR-bitmap read permissions.
294 */
295typedef enum VMXMSREXITREAD
296{
297 /** Reading this MSR causes a VM-exit. */
298 VMXMSREXIT_INTERCEPT_READ = 0xb,
299 /** Reading this MSR does not cause a VM-exit. */
300 VMXMSREXIT_PASSTHRU_READ
301} VMXMSREXITREAD;
302/** Pointer to MSR-bitmap read permissions. */
303typedef VMXMSREXITREAD* PVMXMSREXITREAD;
304
305/**
306 * MSR-bitmap write permissions.
307 */
308typedef enum VMXMSREXITWRITE
309{
310 /** Writing to this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
312 /** Writing to this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_WRITE
314} VMXMSREXITWRITE;
315/** Pointer to MSR-bitmap write permissions. */
316typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
317
318
319/**
320 * VMX VM-exit handler.
321 *
322 * @returns Strict VBox status code (i.e. informational status codes too).
323 * @param pVCpu The cross context virtual CPU structure.
324 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
325 * out-of-sync. Make sure to update the required
326 * fields before using them.
327 * @param pVmxTransient Pointer to the VMX-transient structure.
328 */
329#ifndef HMVMX_USE_FUNCTION_TABLE
330typedef DECLINLINE(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331#else
332typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
333/** Pointer to VM-exit handler. */
334typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
335#endif
336
337/**
338 * VMX VM-exit handler, non-strict status code.
339 *
340 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
341 *
342 * @returns VBox status code, no informational status code returned.
343 * @param pVCpu The cross context virtual CPU structure.
344 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
345 * out-of-sync. Make sure to update the required
346 * fields before using them.
347 * @param pVmxTransient Pointer to the VMX-transient structure.
348 *
349 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
350 * use of that status code will be replaced with VINF_EM_SOMETHING
351 * later when switching over to IEM.
352 */
353#ifndef HMVMX_USE_FUNCTION_TABLE
354typedef DECLINLINE(int) FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
355#else
356typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
357#endif
358
359
360/*********************************************************************************************************************************
361* Internal Functions *
362*********************************************************************************************************************************/
363static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
364static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
365static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
366static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
367 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
368 bool fStepping, uint32_t *puIntState);
369#if HC_ARCH_BITS == 32
370static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
371#endif
372#ifndef HMVMX_USE_FUNCTION_TABLE
373DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
374# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
375# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
376#else
377# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
378# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
379#endif
380
381
382/** @name VM-exit handlers.
383 * @{
384 */
385static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
386static FNVMXEXITHANDLER hmR0VmxExitExtInt;
387static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
388static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
389static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
390static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
391static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
394static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
395static FNVMXEXITHANDLER hmR0VmxExitCpuid;
396static FNVMXEXITHANDLER hmR0VmxExitGetsec;
397static FNVMXEXITHANDLER hmR0VmxExitHlt;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
399static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
400static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
401static FNVMXEXITHANDLER hmR0VmxExitVmcall;
402static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
405static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
406static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
407static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
408static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
409static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
413static FNVMXEXITHANDLER hmR0VmxExitMwait;
414static FNVMXEXITHANDLER hmR0VmxExitMtf;
415static FNVMXEXITHANDLER hmR0VmxExitMonitor;
416static FNVMXEXITHANDLER hmR0VmxExitPause;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
418static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
419static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
420static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
421static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
422static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
423static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
424static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
425static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
427static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
428static FNVMXEXITHANDLER hmR0VmxExitRdrand;
429static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
430/** @} */
431
432static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
433static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
434static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
435static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
436static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
440static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
441
442
443/*********************************************************************************************************************************
444* Global Variables *
445*********************************************************************************************************************************/
446#ifdef HMVMX_USE_FUNCTION_TABLE
447
448/**
449 * VMX_EXIT dispatch table.
450 */
451static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
452{
453 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
454 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
455 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
456 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
457 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
458 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
459 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
460 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
461 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
462 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
463 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
464 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
465 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
466 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
467 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
468 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
469 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
470 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
471 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
472 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
473 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
474 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
475 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
476 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
477 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
478 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
479 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
480 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
481 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
482 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
483 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
484 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
485 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
486 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
487 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
488 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
489 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
490 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
491 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
493 /* 40 UNDEFINED */ hmR0VmxExitPause,
494 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
495 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
496 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
497 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
498 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
500 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
501 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
502 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
503 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
504 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
505 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
506 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
507 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
508 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
509 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
510 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
511 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
512 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
513 /* 60 VMX_EXIT_RESERVED_60 */ hmR0VmxExitErrUndefined,
514 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
515 /* 62 VMX_EXIT_RESERVED_62 */ hmR0VmxExitErrUndefined,
516 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
517 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
518};
519#endif /* HMVMX_USE_FUNCTION_TABLE */
520
521#ifdef VBOX_STRICT
522static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
523{
524 /* 0 */ "(Not Used)",
525 /* 1 */ "VMCALL executed in VMX root operation.",
526 /* 2 */ "VMCLEAR with invalid physical address.",
527 /* 3 */ "VMCLEAR with VMXON pointer.",
528 /* 4 */ "VMLAUNCH with non-clear VMCS.",
529 /* 5 */ "VMRESUME with non-launched VMCS.",
530 /* 6 */ "VMRESUME after VMXOFF",
531 /* 7 */ "VM-entry with invalid control fields.",
532 /* 8 */ "VM-entry with invalid host state fields.",
533 /* 9 */ "VMPTRLD with invalid physical address.",
534 /* 10 */ "VMPTRLD with VMXON pointer.",
535 /* 11 */ "VMPTRLD with incorrect revision identifier.",
536 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
537 /* 13 */ "VMWRITE to read-only VMCS component.",
538 /* 14 */ "(Not Used)",
539 /* 15 */ "VMXON executed in VMX root operation.",
540 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
541 /* 17 */ "VM-entry with non-launched executing VMCS.",
542 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
543 /* 19 */ "VMCALL with non-clear VMCS.",
544 /* 20 */ "VMCALL with invalid VM-exit control fields.",
545 /* 21 */ "(Not Used)",
546 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
547 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
548 /* 24 */ "VMCALL with invalid SMM-monitor features.",
549 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
550 /* 26 */ "VM-entry with events blocked by MOV SS.",
551 /* 27 */ "(Not Used)",
552 /* 28 */ "Invalid operand to INVEPT/INVVPID."
553};
554#endif /* VBOX_STRICT */
555
556
557
558/**
559 * Updates the VM's last error record.
560 *
561 * If there was a VMX instruction error, reads the error data from the VMCS and
562 * updates VCPU's last error record as well.
563 *
564 * @param pVM The cross context VM structure.
565 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
566 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
567 * VERR_VMX_INVALID_VMCS_FIELD.
568 * @param rc The error code.
569 */
570static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
571{
572 AssertPtr(pVM);
573 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
574 || rc == VERR_VMX_UNABLE_TO_START_VM)
575 {
576 AssertPtrReturnVoid(pVCpu);
577 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
578 }
579 pVM->hm.s.lLastError = rc;
580}
581
582
583/**
584 * Reads the VM-entry interruption-information field from the VMCS into the VMX
585 * transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-entry exception error code field from the VMCS into
602 * the VMX transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-entry exception error code field from the VMCS into
619 * the VMX transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 *
624 * @remarks No-long-jump zone!!!
625 */
626DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
629 AssertRCReturn(rc, rc);
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit interruption-information field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVmxTransient Pointer to the VMX transient structure.
640 */
641DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
642{
643 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
644 {
645 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
646 AssertRCReturn(rc, rc);
647 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
648 }
649 return VINF_SUCCESS;
650}
651
652
653/**
654 * Reads the VM-exit interruption error code from the VMCS into the VMX
655 * transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVmxTransient Pointer to the VMX transient structure.
659 */
660DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
661{
662 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
663 {
664 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
665 AssertRCReturn(rc, rc);
666 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
667 }
668 return VINF_SUCCESS;
669}
670
671
672/**
673 * Reads the VM-exit instruction length field from the VMCS into the VMX
674 * transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVmxTransient Pointer to the VMX transient structure.
678 */
679DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
680{
681 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
682 {
683 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
684 AssertRCReturn(rc, rc);
685 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
686 }
687 return VINF_SUCCESS;
688}
689
690
691/**
692 * Reads the VM-exit instruction-information field from the VMCS into
693 * the VMX transient structure.
694 *
695 * @returns VBox status code.
696 * @param pVmxTransient Pointer to the VMX transient structure.
697 */
698DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
699{
700 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
701 {
702 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
703 AssertRCReturn(rc, rc);
704 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
705 }
706 return VINF_SUCCESS;
707}
708
709
710/**
711 * Reads the exit code qualification from the VMCS into the VMX transient
712 * structure.
713 *
714 * @returns VBox status code.
715 * @param pVCpu The cross context virtual CPU structure of the
716 * calling EMT. (Required for the VMCS cache case.)
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
722 {
723 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the IDT-vectoring information field from the VMCS into the VMX
733 * transient structure.
734 *
735 * @returns VBox status code.
736 * @param pVmxTransient Pointer to the VMX transient structure.
737 *
738 * @remarks No-long-jump zone!!!
739 */
740DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
743 {
744 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring error code from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 */
759DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
760{
761 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
762 {
763 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
764 AssertRCReturn(rc, rc);
765 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
766 }
767 return VINF_SUCCESS;
768}
769
770
771/**
772 * Enters VMX root mode operation on the current CPU.
773 *
774 * @returns VBox status code.
775 * @param pVM The cross context VM structure. Can be
776 * NULL, after a resume.
777 * @param HCPhysCpuPage Physical address of the VMXON region.
778 * @param pvCpuPage Pointer to the VMXON region.
779 */
780static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
781{
782 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
783 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
784 Assert(pvCpuPage);
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 if (pVM)
788 {
789 /* Write the VMCS revision dword to the VMXON region. */
790 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
791 }
792
793 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
794 RTCCUINTREG fEFlags = ASMIntDisableFlags();
795
796 /* Enable the VMX bit in CR4 if necessary. */
797 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
798
799 /* Enter VMX root mode. */
800 int rc = VMXEnable(HCPhysCpuPage);
801 if (RT_FAILURE(rc))
802 {
803 if (!(uOldCr4 & X86_CR4_VMXE))
804 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
805
806 if (pVM)
807 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
808 }
809
810 /* Restore interrupts. */
811 ASMSetFlags(fEFlags);
812 return rc;
813}
814
815
816/**
817 * Exits VMX root mode operation on the current CPU.
818 *
819 * @returns VBox status code.
820 */
821static int hmR0VmxLeaveRootMode(void)
822{
823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
824
825 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
826 RTCCUINTREG fEFlags = ASMIntDisableFlags();
827
828 /* If we're for some reason not in VMX root mode, then don't leave it. */
829 RTCCUINTREG uHostCR4 = ASMGetCR4();
830
831 int rc;
832 if (uHostCR4 & X86_CR4_VMXE)
833 {
834 /* Exit VMX root mode and clear the VMX bit in CR4. */
835 VMXDisable();
836 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
837 rc = VINF_SUCCESS;
838 }
839 else
840 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
841
842 /* Restore interrupts. */
843 ASMSetFlags(fEFlags);
844 return rc;
845}
846
847
848/**
849 * Allocates and maps one physically contiguous page. The allocated page is
850 * zero'd out. (Used by various VT-x structures).
851 *
852 * @returns IPRT status code.
853 * @param pMemObj Pointer to the ring-0 memory object.
854 * @param ppVirt Where to store the virtual address of the
855 * allocation.
856 * @param pHCPhys Where to store the physical address of the
857 * allocation.
858 */
859DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
860{
861 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
862 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
863 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
864
865 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
866 if (RT_FAILURE(rc))
867 return rc;
868 *ppVirt = RTR0MemObjAddress(*pMemObj);
869 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
870 ASMMemZero32(*ppVirt, PAGE_SIZE);
871 return VINF_SUCCESS;
872}
873
874
875/**
876 * Frees and unmaps an allocated physical page.
877 *
878 * @param pMemObj Pointer to the ring-0 memory object.
879 * @param ppVirt Where to re-initialize the virtual address of
880 * allocation as 0.
881 * @param pHCPhys Where to re-initialize the physical address of the
882 * allocation as 0.
883 */
884DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
885{
886 AssertPtr(pMemObj);
887 AssertPtr(ppVirt);
888 AssertPtr(pHCPhys);
889 if (*pMemObj != NIL_RTR0MEMOBJ)
890 {
891 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
892 AssertRC(rc);
893 *pMemObj = NIL_RTR0MEMOBJ;
894 *ppVirt = 0;
895 *pHCPhys = 0;
896 }
897}
898
899
900/**
901 * Worker function to free VT-x related structures.
902 *
903 * @returns IPRT status code.
904 * @param pVM The cross context VM structure.
905 */
906static void hmR0VmxStructsFree(PVM pVM)
907{
908 for (VMCPUID i = 0; i < pVM->cCpus; i++)
909 {
910 PVMCPU pVCpu = &pVM->aCpus[i];
911 AssertPtr(pVCpu);
912
913 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
914 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
915
916 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
917 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
918
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
921 }
922
923 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
924#ifdef VBOX_WITH_CRASHDUMP_MAGIC
925 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
926#endif
927}
928
929
930/**
931 * Worker function to allocate VT-x related VM structures.
932 *
933 * @returns IPRT status code.
934 * @param pVM The cross context VM structure.
935 */
936static int hmR0VmxStructsAlloc(PVM pVM)
937{
938 /*
939 * Initialize members up-front so we can cleanup properly on allocation failure.
940 */
941#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
942 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
943 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
944 pVM->hm.s.vmx.HCPhys##a_Name = 0;
945
946#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
947 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
948 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
949 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
950
951#ifdef VBOX_WITH_CRASHDUMP_MAGIC
952 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
953#endif
954 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
955
956 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
957 for (VMCPUID i = 0; i < pVM->cCpus; i++)
958 {
959 PVMCPU pVCpu = &pVM->aCpus[i];
960 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
1009 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1010 {
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1012 &pVCpu->hm.s.vmx.HCPhysVirtApic);
1013 if (RT_FAILURE(rc))
1014 goto cleanup;
1015 }
1016
1017 /*
1018 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1019 * transparent accesses of specific MSRs.
1020 *
1021 * If the condition for enabling MSR bitmaps changes here, don't forget to
1022 * update HMAreMsrBitmapsAvailable().
1023 */
1024 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1025 {
1026 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1027 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1028 if (RT_FAILURE(rc))
1029 goto cleanup;
1030 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1031 }
1032
1033 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037
1038 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1039 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1040 if (RT_FAILURE(rc))
1041 goto cleanup;
1042 }
1043
1044 return VINF_SUCCESS;
1045
1046cleanup:
1047 hmR0VmxStructsFree(pVM);
1048 return rc;
1049}
1050
1051
1052/**
1053 * Does global VT-x initialization (called during module initialization).
1054 *
1055 * @returns VBox status code.
1056 */
1057VMMR0DECL(int) VMXR0GlobalInit(void)
1058{
1059#ifdef HMVMX_USE_FUNCTION_TABLE
1060 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1061# ifdef VBOX_STRICT
1062 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1063 Assert(g_apfnVMExitHandlers[i]);
1064# endif
1065#endif
1066 return VINF_SUCCESS;
1067}
1068
1069
1070/**
1071 * Does global VT-x termination (called during module termination).
1072 */
1073VMMR0DECL(void) VMXR0GlobalTerm()
1074{
1075 /* Nothing to do currently. */
1076}
1077
1078
1079/**
1080 * Sets up and activates VT-x on the current CPU.
1081 *
1082 * @returns VBox status code.
1083 * @param pCpu Pointer to the global CPU info struct.
1084 * @param pVM The cross context VM structure. Can be
1085 * NULL after a host resume operation.
1086 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1087 * fEnabledByHost is @c true).
1088 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1089 * @a fEnabledByHost is @c true).
1090 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1091 * enable VT-x on the host.
1092 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1093 */
1094VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1095 void *pvMsrs)
1096{
1097 Assert(pCpu);
1098 Assert(pvMsrs);
1099 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1100
1101 /* Enable VT-x if it's not already enabled by the host. */
1102 if (!fEnabledByHost)
1103 {
1104 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1105 if (RT_FAILURE(rc))
1106 return rc;
1107 }
1108
1109 /*
1110 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1111 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1112 */
1113 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1114 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1115 {
1116 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1117 pCpu->fFlushAsidBeforeUse = false;
1118 }
1119 else
1120 pCpu->fFlushAsidBeforeUse = true;
1121
1122 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1123 ++pCpu->cTlbFlushes;
1124
1125 return VINF_SUCCESS;
1126}
1127
1128
1129/**
1130 * Deactivates VT-x on the current CPU.
1131 *
1132 * @returns VBox status code.
1133 * @param pCpu Pointer to the global CPU info struct.
1134 * @param pvCpuPage Pointer to the VMXON region.
1135 * @param HCPhysCpuPage Physical address of the VMXON region.
1136 *
1137 * @remarks This function should never be called when SUPR0EnableVTx() or
1138 * similar was used to enable VT-x on the host.
1139 */
1140VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1141{
1142 NOREF(pCpu);
1143 NOREF(pvCpuPage);
1144 NOREF(HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST));
1476 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477 {
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#endif
1516 return false;
1517}
1518
1519
1520/**
1521 * Saves a set of guest MSRs back into the guest-CPU context.
1522 *
1523 * @param pVCpu The cross context virtual CPU structure.
1524 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1525 * out-of-sync. Make sure to update the required fields
1526 * before using them.
1527 *
1528 * @remarks No-long-jump zone!!!
1529 */
1530static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1531{
1532 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1533 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1534
1535 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1536 {
1537 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1538#if HC_ARCH_BITS == 64
1539 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1540 {
1541 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1542 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1543 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1544 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1545 }
1546#endif
1547 }
1548}
1549
1550
1551/**
1552 * Loads a set of guests MSRs to allow read/passthru to the guest.
1553 *
1554 * The name of this function is slightly confusing. This function does NOT
1555 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1556 * common prefix for functions dealing with "lazy restoration" of the shared
1557 * MSRs.
1558 *
1559 * @param pVCpu The cross context virtual CPU structure.
1560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1561 * out-of-sync. Make sure to update the required fields
1562 * before using them.
1563 *
1564 * @remarks No-long-jump zone!!!
1565 */
1566static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1567{
1568 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1569 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1570
1571#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1572 do { \
1573 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1574 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1575 else \
1576 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1577 } while (0)
1578
1579 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1580 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
1581 {
1582#if HC_ARCH_BITS == 64
1583 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1584 {
1585 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1586 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1587 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1588 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1589 }
1590#endif
1591 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1592 }
1593
1594#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1595}
1596
1597
1598/**
1599 * Performs lazy restoration of the set of host MSRs if they were previously
1600 * loaded with guest MSR values.
1601 *
1602 * @param pVCpu The cross context virtual CPU structure.
1603 *
1604 * @remarks No-long-jump zone!!!
1605 * @remarks The guest MSRs should have been saved back into the guest-CPU
1606 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1607 */
1608static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1609{
1610 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1611 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1612
1613 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1614 {
1615 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1616#if HC_ARCH_BITS == 64
1617 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1618 {
1619 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1620 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1621 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1622 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1623 }
1624#endif
1625 }
1626 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1627}
1628
1629
1630/**
1631 * Verifies that our cached values of the VMCS controls are all
1632 * consistent with what's actually present in the VMCS.
1633 *
1634 * @returns VBox status code.
1635 * @param pVCpu The cross context virtual CPU structure.
1636 */
1637static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1638{
1639 uint32_t u32Val;
1640 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1643 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1648 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1653 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1654
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1658 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1659
1660 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661 {
1662 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1663 AssertRCReturn(rc, rc);
1664 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1665 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1666 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1667 }
1668
1669 return VINF_SUCCESS;
1670}
1671
1672
1673#ifdef VBOX_STRICT
1674/**
1675 * Verifies that our cached host EFER value has not changed
1676 * since we cached it.
1677 *
1678 * @param pVCpu The cross context virtual CPU structure.
1679 */
1680static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1681{
1682 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1683
1684 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1685 {
1686 uint64_t u64Val;
1687 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1688 AssertRC(rc);
1689
1690 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1691 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1692 }
1693}
1694
1695
1696/**
1697 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1698 * VMCS are correct.
1699 *
1700 * @param pVCpu The cross context virtual CPU structure.
1701 */
1702static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1703{
1704 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1705
1706 /* Verify MSR counts in the VMCS are what we think it should be. */
1707 uint32_t cMsrs;
1708 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1709 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1710
1711 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1712 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1713
1714 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1715 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1716
1717 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1718 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1719 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1720 {
1721 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1722 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1723 pGuestMsr->u32Msr, cMsrs));
1724
1725 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1726 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1727 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1728
1729 /* Verify that the permissions are as expected in the MSR bitmap. */
1730 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1731 {
1732 VMXMSREXITREAD enmRead;
1733 VMXMSREXITWRITE enmWrite;
1734 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1735 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1736 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1739 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1740 }
1741 else
1742 {
1743 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1744 pGuestMsr->u32Msr, cMsrs));
1745 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1746 pGuestMsr->u32Msr, cMsrs));
1747 }
1748 }
1749 }
1750}
1751#endif /* VBOX_STRICT */
1752
1753
1754/**
1755 * Flushes the TLB using EPT.
1756 *
1757 * @returns VBox status code.
1758 * @param pVCpu The cross context virtual CPU structure of the calling
1759 * EMT. Can be NULL depending on @a enmFlush.
1760 * @param enmFlush Type of flush.
1761 *
1762 * @remarks Caller is responsible for making sure this function is called only
1763 * when NestedPaging is supported and providing @a enmFlush that is
1764 * supported by the CPU.
1765 * @remarks Can be called with interrupts disabled.
1766 */
1767static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1768{
1769 uint64_t au64Descriptor[2];
1770 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1771 au64Descriptor[0] = 0;
1772 else
1773 {
1774 Assert(pVCpu);
1775 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1776 }
1777 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1778
1779 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1780 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1781 rc));
1782 if ( RT_SUCCESS(rc)
1783 && pVCpu)
1784 {
1785 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1786 }
1787}
1788
1789
1790/**
1791 * Flushes the TLB using VPID.
1792 *
1793 * @returns VBox status code.
1794 * @param pVM The cross context VM structure.
1795 * @param pVCpu The cross context virtual CPU structure of the calling
1796 * EMT. Can be NULL depending on @a enmFlush.
1797 * @param enmFlush Type of flush.
1798 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1799 * on @a enmFlush).
1800 *
1801 * @remarks Can be called with interrupts disabled.
1802 */
1803static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1804{
1805 NOREF(pVM);
1806 AssertPtr(pVM);
1807 Assert(pVM->hm.s.vmx.fVpid);
1808
1809 uint64_t au64Descriptor[2];
1810 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1811 {
1812 au64Descriptor[0] = 0;
1813 au64Descriptor[1] = 0;
1814 }
1815 else
1816 {
1817 AssertPtr(pVCpu);
1818 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1819 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1820 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1821 au64Descriptor[1] = GCPtr;
1822 }
1823
1824 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1825 AssertMsg(rc == VINF_SUCCESS,
1826 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1827 if ( RT_SUCCESS(rc)
1828 && pVCpu)
1829 {
1830 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1831 }
1832}
1833
1834
1835/**
1836 * Invalidates a guest page by guest virtual address. Only relevant for
1837 * EPT/VPID, otherwise there is nothing really to invalidate.
1838 *
1839 * @returns VBox status code.
1840 * @param pVM The cross context VM structure.
1841 * @param pVCpu The cross context virtual CPU structure.
1842 * @param GCVirt Guest virtual address of the page to invalidate.
1843 */
1844VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1845{
1846 AssertPtr(pVM);
1847 AssertPtr(pVCpu);
1848 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1849
1850 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1851 if (!fFlushPending)
1852 {
1853 /*
1854 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1855 * See @bugref{6043} and @bugref{6177}.
1856 *
1857 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1858 * function maybe called in a loop with individual addresses.
1859 */
1860 if (pVM->hm.s.vmx.fVpid)
1861 {
1862 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1863 {
1864 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1880 * otherwise there is nothing really to invalidate.
1881 *
1882 * @returns VBox status code.
1883 * @param pVM The cross context VM structure.
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param GCPhys Guest physical address of the page to invalidate.
1886 */
1887VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1888{
1889 NOREF(pVM); NOREF(GCPhys);
1890 LogFlowFunc(("%RGp\n", GCPhys));
1891
1892 /*
1893 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1894 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1895 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1896 */
1897 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1898 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1899 return VINF_SUCCESS;
1900}
1901
1902
1903/**
1904 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1905 * case where neither EPT nor VPID is supported by the CPU.
1906 *
1907 * @param pVM The cross context VM structure.
1908 * @param pVCpu The cross context virtual CPU structure.
1909 * @param pCpu Pointer to the global HM struct.
1910 *
1911 * @remarks Called with interrupts disabled.
1912 */
1913static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1914{
1915 AssertPtr(pVCpu);
1916 AssertPtr(pCpu);
1917 NOREF(pVM);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVM The cross context VM structure.
1933 * @param pVCpu The cross context virtual CPU structure.
1934 * @param pCpu Pointer to the global HM CPU struct.
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in
1936 * Intel's nomenclature. The reason is, to avoid confusion in compare
1937 * statements since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pVM);
1956 AssertPtr(pCpu);
1957 AssertPtr(pVCpu);
1958 Assert(pCpu->idCpu != NIL_RTCPUID);
1959
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1966 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1967 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1968 */
1969 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1970 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1971 {
1972 ++pCpu->uCurrentAsid;
1973 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1974 {
1975 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1976 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1977 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1978 }
1979
1980 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1981 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1982 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1983
1984 /*
1985 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1986 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1987 */
1988 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1989 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1990 HMVMX_SET_TAGGED_TLB_FLUSHED();
1991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1992 }
1993
1994 /* Check for explicit TLB flushes. */
1995 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1996 {
1997 /*
1998 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1999 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2000 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2001 * but not guest-physical mappings.
2002 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2003 */
2004 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2005 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2006 HMVMX_SET_TAGGED_TLB_FLUSHED();
2007 }
2008
2009 pVCpu->hm.s.fForceTLBFlush = false;
2010 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2011
2012 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2013 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2014 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2015 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2016 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2017 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2018 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2019 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2021
2022 /* Update VMCS with the VPID. */
2023 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2024 AssertRC(rc);
2025
2026#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2027}
2028
2029
2030/**
2031 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2032 *
2033 * @returns VBox status code.
2034 * @param pVM The cross context VM structure.
2035 * @param pVCpu The cross context virtual CPU structure.
2036 * @param pCpu Pointer to the global HM CPU struct.
2037 *
2038 * @remarks Called with interrupts disabled.
2039 */
2040static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2041{
2042 AssertPtr(pVM);
2043 AssertPtr(pVCpu);
2044 AssertPtr(pCpu);
2045 Assert(pCpu->idCpu != NIL_RTCPUID);
2046 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2047 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2048
2049 /*
2050 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2051 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2052 */
2053 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2054 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2055 {
2056 pVCpu->hm.s.fForceTLBFlush = true;
2057 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2058 }
2059
2060 /* Check for explicit TLB flushes. */
2061 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2062 {
2063 pVCpu->hm.s.fForceTLBFlush = true;
2064 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2065 }
2066
2067 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2068 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2069
2070 if (pVCpu->hm.s.fForceTLBFlush)
2071 {
2072 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2073 pVCpu->hm.s.fForceTLBFlush = false;
2074 }
2075}
2076
2077
2078/**
2079 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2080 *
2081 * @returns VBox status code.
2082 * @param pVM The cross context VM structure.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVM);
2091 AssertPtr(pVCpu);
2092 AssertPtr(pCpu);
2093 Assert(pCpu->idCpu != NIL_RTCPUID);
2094 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2095 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2096
2097 /*
2098 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2099 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2100 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2114 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2115 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2116 */
2117 pVCpu->hm.s.fForceTLBFlush = true;
2118 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2119 }
2120
2121 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2122 if (pVCpu->hm.s.fForceTLBFlush)
2123 {
2124 ++pCpu->uCurrentAsid;
2125 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2126 {
2127 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2128 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2129 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2130 }
2131
2132 pVCpu->hm.s.fForceTLBFlush = false;
2133 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2134 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2135 if (pCpu->fFlushAsidBeforeUse)
2136 {
2137 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2139 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2140 {
2141 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2142 pCpu->fFlushAsidBeforeUse = false;
2143 }
2144 else
2145 {
2146 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2147 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2148 }
2149 }
2150 }
2151
2152 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2153 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2154 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2155 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2156 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2157 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2158 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2159
2160 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2161 AssertRC(rc);
2162}
2163
2164
2165/**
2166 * Flushes the guest TLB entry based on CPU capabilities.
2167 *
2168 * @param pVCpu The cross context virtual CPU structure.
2169 * @param pCpu Pointer to the global HM CPU struct.
2170 */
2171DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2172{
2173#ifdef HMVMX_ALWAYS_FLUSH_TLB
2174 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2175#endif
2176 PVM pVM = pVCpu->CTX_SUFF(pVM);
2177 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2178 {
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2182 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2183 default:
2184 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2185 break;
2186 }
2187
2188 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2189}
2190
2191
2192/**
2193 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2194 * TLB entries from the host TLB before VM-entry.
2195 *
2196 * @returns VBox status code.
2197 * @param pVM The cross context VM structure.
2198 */
2199static int hmR0VmxSetupTaggedTlb(PVM pVM)
2200{
2201 /*
2202 * Determine optimal flush type for Nested Paging.
2203 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2204 * guest execution (see hmR3InitFinalizeR0()).
2205 */
2206 if (pVM->hm.s.fNestedPaging)
2207 {
2208 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2212 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2213 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221
2222 /* Make sure the write-back cacheable memory type for EPT is supported. */
2223 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2224 {
2225 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2226 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2228 }
2229
2230 /* EPT requires a page-walk length of 4. */
2231 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2232 {
2233 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2234 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238 else
2239 {
2240 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2241 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2242 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2243 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2244 }
2245 }
2246
2247 /*
2248 * Determine optimal flush type for VPID.
2249 */
2250 if (pVM->hm.s.vmx.fVpid)
2251 {
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2253 {
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2255 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2256 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2257 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2258 else
2259 {
2260 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2262 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2264 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269 else
2270 {
2271 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2272 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277
2278 /*
2279 * Setup the handler for flushing tagged-TLBs.
2280 */
2281 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2282 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2283 else if (pVM->hm.s.fNestedPaging)
2284 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2285 else if (pVM->hm.s.vmx.fVpid)
2286 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2287 else
2288 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2289 return VINF_SUCCESS;
2290}
2291
2292
2293/**
2294 * Sets up pin-based VM-execution controls in the VMCS.
2295 *
2296 * @returns VBox status code.
2297 * @param pVM The cross context VM structure.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 */
2300static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2301{
2302 AssertPtr(pVM);
2303 AssertPtr(pVCpu);
2304
2305 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2306 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2307
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2309 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2310
2311 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2312 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2313
2314 /* Enable the VMX preemption timer. */
2315 if (pVM->hm.s.vmx.fUsePreemptTimer)
2316 {
2317 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2318 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2319 }
2320
2321#ifdef VBOX_WITH_NEW_APIC
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331#endif
2332
2333 if ((val & zap) != val)
2334 {
2335 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2336 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2337 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2338 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2339 }
2340
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2342 AssertRCReturn(rc, rc);
2343
2344 pVCpu->hm.s.vmx.u32PinCtls = val;
2345 return rc;
2346}
2347
2348
2349/**
2350 * Sets up processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVM The cross context VM structure.
2354 * @param pVCpu The cross context virtual CPU structure.
2355 */
2356static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2357{
2358 AssertPtr(pVM);
2359 AssertPtr(pVCpu);
2360
2361 int rc = VERR_INTERNAL_ERROR_5;
2362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2364
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2368 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2369 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2370 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2372
2373 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2374 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2375 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2376 {
2377 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2378 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2379 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2380 }
2381
2382 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2383 if (!pVM->hm.s.fNestedPaging)
2384 {
2385 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2386 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2387 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2389 }
2390
2391 /* Use TPR shadowing if supported by the CPU. */
2392 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2393 {
2394 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2396 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2397 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2398 AssertRCReturn(rc, rc);
2399
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2401 /* CR8 writes cause a VM-exit based on TPR threshold. */
2402 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2403 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2404 }
2405 else
2406 {
2407 /*
2408 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2409 * Set this control only for 64-bit guests.
2410 */
2411 if (pVM->hm.s.fAllow64BitGuests)
2412 {
2413 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2414 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2415 }
2416 }
2417
2418 /* Use MSR-bitmaps if supported by the CPU. */
2419 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2420 {
2421 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2422
2423 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2424 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2425 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2426 AssertRCReturn(rc, rc);
2427
2428 /*
2429 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2430 * automatically using dedicated fields in the VMCS.
2431 */
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2434 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2435 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2436 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2437
2438#if HC_ARCH_BITS == 64
2439 /*
2440 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2441 */
2442 if (pVM->hm.s.fAllow64BitGuests)
2443 {
2444 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2445 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2446 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2448 }
2449#endif
2450 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2451 }
2452
2453 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2454 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2455 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2456
2457 if ((val & zap) != val)
2458 {
2459 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2460 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2461 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2462 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2463 }
2464
2465 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2466 AssertRCReturn(rc, rc);
2467
2468 pVCpu->hm.s.vmx.u32ProcCtls = val;
2469
2470 /*
2471 * Secondary processor-based VM-execution controls.
2472 */
2473 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2474 {
2475 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2476 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2477
2478 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2479 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2480
2481 if (pVM->hm.s.fNestedPaging)
2482 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2483 else
2484 {
2485 /*
2486 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2487 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2488 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2489 */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2492 }
2493
2494 if (pVM->hm.s.vmx.fVpid)
2495 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2496
2497 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2498 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2499
2500#ifdef VBOX_WITH_NEW_APIC
2501#if 0
2502 if (pVM->hm.s.fVirtApicRegs)
2503 {
2504 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2505 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2506
2507 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2509 }
2510#endif
2511#endif
2512
2513 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2514 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2515 * done dynamically. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2517 {
2518 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2519 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2520 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2521 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2522 AssertRCReturn(rc, rc);
2523 }
2524
2525 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2526 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2527
2528 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2529 && pVM->hm.s.vmx.cPleGapTicks
2530 && pVM->hm.s.vmx.cPleWindowTicks)
2531 {
2532 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2533
2534 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2535 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2536 AssertRCReturn(rc, rc);
2537 }
2538
2539 if ((val & zap) != val)
2540 {
2541 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2542 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2543 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2544 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2545 }
2546
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2548 AssertRCReturn(rc, rc);
2549
2550 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2551 }
2552 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2555 "available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM The cross context VM structure.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2573{
2574 NOREF(pVM);
2575 AssertPtr(pVM);
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /** @todo Explore possibility of using IO-bitmaps. */
2595 /* All IO & IOIO instructions cause VM-exits. */
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2598
2599 /* Initialize the MSR-bitmap area. */
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2602 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2603 AssertRCReturn(rc, rc);
2604#endif
2605
2606 /* Setup MSR auto-load/store area. */
2607 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2609 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2614 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2616 AssertRCReturn(rc, rc);
2617
2618 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2619 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2620 AssertRCReturn(rc, rc);
2621
2622 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2623#if 0
2624 /* Setup debug controls */
2625 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2626 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2627 AssertRCReturn(rc, rc);
2628#endif
2629
2630 return rc;
2631}
2632
2633
2634/**
2635 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM The cross context VM structure.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2642{
2643 AssertPtr(pVM);
2644 AssertPtr(pVCpu);
2645
2646 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2647
2648 uint32_t u32XcptBitmap = pVCpu->hm.s.fGIMTrapXcptUD ? RT_BIT(X86_XCPT_UD) : 0;
2649
2650 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2652
2653 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2654 and writes, and because recursive #DBs can cause the CPU hang, we must always
2655 intercept #DB. */
2656 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2657
2658 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2659 if (!pVM->hm.s.fNestedPaging)
2660 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2661
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2664 AssertRCReturn(rc, rc);
2665 return rc;
2666}
2667
2668
2669/**
2670 * Sets up the initial guest-state mask. The guest-state mask is consulted
2671 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2672 * for the nested virtualization case (as it would cause a VM-exit).
2673 *
2674 * @param pVCpu The cross context virtual CPU structure.
2675 */
2676static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2677{
2678 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2679 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2740 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2788 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2789 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2790
2791 /* Set revision dword at the beginning of the VMCS structure. */
2792 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2793
2794 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2795 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2796 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2797 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2798
2799 /* Load this VMCS as the current VMCS. */
2800 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2802 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2811
2812 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2814 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2815
2816 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2817 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2818 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2819
2820 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824#if HC_ARCH_BITS == 32
2825 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2826 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2827 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2828#endif
2829
2830 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2831 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2832 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2833 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2834
2835 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2836
2837 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2838 }
2839
2840 return VINF_SUCCESS;
2841}
2842
2843
2844/**
2845 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2846 * the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVM The cross context VM structure.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2853{
2854 NOREF(pVM); NOREF(pVCpu);
2855
2856 RTCCUINTREG uReg = ASMGetCR0();
2857 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2858 AssertRCReturn(rc, rc);
2859
2860 uReg = ASMGetCR3();
2861 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM The cross context VM structure.
2901 * @param pVCpu The cross context virtual CPU structure.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 int rc = VERR_INTERNAL_ERROR_5;
2906
2907#if HC_ARCH_BITS == 64
2908 /*
2909 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2910 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2911 */
2912 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2913 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2914#endif
2915
2916 /*
2917 * Host DS, ES, FS and GS segment registers.
2918 */
2919#if HC_ARCH_BITS == 64
2920 RTSEL uSelDS = ASMGetDS();
2921 RTSEL uSelES = ASMGetES();
2922 RTSEL uSelFS = ASMGetFS();
2923 RTSEL uSelGS = ASMGetGS();
2924#else
2925 RTSEL uSelDS = 0;
2926 RTSEL uSelES = 0;
2927 RTSEL uSelFS = 0;
2928 RTSEL uSelGS = 0;
2929#endif
2930
2931 /* Recalculate which host-state bits need to be manually restored. */
2932 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2933
2934 /*
2935 * Host CS and SS segment registers.
2936 */
2937 RTSEL uSelCS = ASMGetCS();
2938 RTSEL uSelSS = ASMGetSS();
2939
2940 /*
2941 * Host TR segment register.
2942 */
2943 RTSEL uSelTR = ASMGetTR();
2944
2945#if HC_ARCH_BITS == 64
2946 /*
2947 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2948 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2949 */
2950 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2951 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2952 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2953 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2954# undef VMXLOCAL_ADJUST_HOST_SEG
2955#endif
2956
2957 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2958 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2959 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2960 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2961 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2962 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2963 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2964 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2965 Assert(uSelCS);
2966 Assert(uSelTR);
2967
2968 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2969#if 0
2970 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2971 Assert(uSelSS != 0);
2972#endif
2973
2974 /* Write these host selector fields into the host-state area in the VMCS. */
2975 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2976 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2977#if HC_ARCH_BITS == 64
2978 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2979 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2980 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2981 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2982#else
2983 NOREF(uSelDS);
2984 NOREF(uSelES);
2985 NOREF(uSelFS);
2986 NOREF(uSelGS);
2987#endif
2988 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2989 AssertRCReturn(rc, rc);
2990
2991 /*
2992 * Host GDTR and IDTR.
2993 */
2994 RTGDTR Gdtr;
2995 RTIDTR Idtr;
2996 RT_ZERO(Gdtr);
2997 RT_ZERO(Idtr);
2998 ASMGetGDTR(&Gdtr);
2999 ASMGetIDTR(&Idtr);
3000 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3001 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3002 AssertRCReturn(rc, rc);
3003
3004#if HC_ARCH_BITS == 64
3005 /*
3006 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3007 * maximum limit (0xffff) on every VM-exit.
3008 */
3009 if (Gdtr.cbGdt != 0xffff)
3010 {
3011 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3012 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3013 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3014 }
3015
3016 /*
3017 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3018 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3019 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3020 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3021 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3022 * hosts where we are pretty sure it won't cause trouble.
3023 */
3024# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3025 if (Idtr.cbIdt < 0x0fff)
3026# else
3027 if (Idtr.cbIdt != 0xffff)
3028# endif
3029 {
3030 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3031 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3032 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3033 }
3034#endif
3035
3036 /*
3037 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3038 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3039 */
3040 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3041 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3042 VERR_VMX_INVALID_HOST_STATE);
3043
3044 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3045#if HC_ARCH_BITS == 64
3046 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3047
3048 /*
3049 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3050 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3051 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3052 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3053 *
3054 * [1] See Intel spec. 3.5 "System Descriptor Types".
3055 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3056 */
3057 Assert(pDesc->System.u4Type == 11);
3058 if ( pDesc->System.u16LimitLow != 0x67
3059 || pDesc->System.u4LimitHigh)
3060 {
3061 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3062 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3063 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3064 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3065 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3066
3067 /* Store the GDTR here as we need it while restoring TR. */
3068 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3069 }
3070#else
3071 NOREF(pVM);
3072 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3073#endif
3074 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /*
3078 * Host FS base and GS base.
3079 */
3080#if HC_ARCH_BITS == 64
3081 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3082 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3083 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3084 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3085 AssertRCReturn(rc, rc);
3086
3087 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3088 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3089 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3090 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3091 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3092#endif
3093 return rc;
3094}
3095
3096
3097/**
3098 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3099 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3100 * the host after every successful VM-exit.
3101 *
3102 * @returns VBox status code.
3103 * @param pVM The cross context VM structure.
3104 * @param pVCpu The cross context virtual CPU structure.
3105 *
3106 * @remarks No-long-jump zone!!!
3107 */
3108DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3109{
3110 NOREF(pVM);
3111
3112 AssertPtr(pVCpu);
3113 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3114
3115 /*
3116 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3117 * rather than swapping them on every VM-entry.
3118 */
3119 hmR0VmxLazySaveHostMsrs(pVCpu);
3120
3121 /*
3122 * Host Sysenter MSRs.
3123 */
3124 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3125#if HC_ARCH_BITS == 32
3126 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3127 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3128#else
3129 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3130 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3131#endif
3132 AssertRCReturn(rc, rc);
3133
3134 /*
3135 * Host EFER MSR.
3136 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3137 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3138 */
3139 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3140 {
3141 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3142 AssertRCReturn(rc, rc);
3143 }
3144
3145 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3146 * hmR0VmxLoadGuestExitCtls() !! */
3147
3148 return rc;
3149}
3150
3151
3152/**
3153 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3154 *
3155 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3156 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3157 * hmR0VMxLoadGuestEntryCtls().
3158 *
3159 * @returns true if we need to load guest EFER, false otherwise.
3160 * @param pVCpu The cross context virtual CPU structure.
3161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3162 * out-of-sync. Make sure to update the required fields
3163 * before using them.
3164 *
3165 * @remarks Requires EFER, CR4.
3166 * @remarks No-long-jump zone!!!
3167 */
3168static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3169{
3170#ifdef HMVMX_ALWAYS_SWAP_EFER
3171 return true;
3172#endif
3173
3174#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3175 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3176 if (CPUMIsGuestInLongMode(pVCpu))
3177 return false;
3178#endif
3179
3180 PVM pVM = pVCpu->CTX_SUFF(pVM);
3181 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3182 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3183
3184 /*
3185 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3186 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3187 */
3188 if ( CPUMIsGuestInLongMode(pVCpu)
3189 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3190 {
3191 return true;
3192 }
3193
3194 /*
3195 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3196 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3197 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3198 */
3199 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3200 && (pMixedCtx->cr0 & X86_CR0_PG)
3201 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3202 {
3203 /* Assert that host is PAE capable. */
3204 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3205 return true;
3206 }
3207
3208 /** @todo Check the latest Intel spec. for any other bits,
3209 * like SMEP/SMAP? */
3210 return false;
3211}
3212
3213
3214/**
3215 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3216 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3217 * controls".
3218 *
3219 * @returns VBox status code.
3220 * @param pVCpu The cross context virtual CPU structure.
3221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3222 * out-of-sync. Make sure to update the required fields
3223 * before using them.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3229{
3230 int rc = VINF_SUCCESS;
3231 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3232 {
3233 PVM pVM = pVCpu->CTX_SUFF(pVM);
3234 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3235 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3236
3237 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3238 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3239
3240 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3241 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3242 {
3243 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3244 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3245 }
3246 else
3247 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3248
3249 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3250 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3251 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3252 {
3253 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3254 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3255 }
3256
3257 /*
3258 * The following should -not- be set (since we're not in SMM mode):
3259 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3260 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3261 */
3262
3263 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3264 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3265
3266 if ((val & zap) != val)
3267 {
3268 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3269 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3270 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3271 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3272 }
3273
3274 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3275 AssertRCReturn(rc, rc);
3276
3277 pVCpu->hm.s.vmx.u32EntryCtls = val;
3278 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3279 }
3280 return rc;
3281}
3282
3283
3284/**
3285 * Sets up the VM-exit controls in the VMCS.
3286 *
3287 * @returns VBox status code.
3288 * @param pVCpu The cross context virtual CPU structure.
3289 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3290 * out-of-sync. Make sure to update the required fields
3291 * before using them.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3296{
3297 NOREF(pMixedCtx);
3298
3299 int rc = VINF_SUCCESS;
3300 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3301 {
3302 PVM pVM = pVCpu->CTX_SUFF(pVM);
3303 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3304 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3305
3306 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3307 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3308
3309 /*
3310 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3311 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3312 */
3313#if HC_ARCH_BITS == 64
3314 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3315 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3316#else
3317 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3318 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3319 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3320 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3321 {
3322 /* The switcher returns to long mode, EFER is managed by the switcher. */
3323 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3324 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3325 }
3326 else
3327 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3328#endif
3329
3330 /* If the newer VMCS fields for managing EFER exists, use it. */
3331 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3332 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3333 {
3334 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3335 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3336 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3337 }
3338
3339 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3340 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3341
3342 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3343 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3344 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3345
3346 if ( pVM->hm.s.vmx.fUsePreemptTimer
3347 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3348 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3349
3350 if ((val & zap) != val)
3351 {
3352 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3353 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3354 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3355 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3356 }
3357
3358 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3359 AssertRCReturn(rc, rc);
3360
3361 pVCpu->hm.s.vmx.u32ExitCtls = val;
3362 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3363 }
3364 return rc;
3365}
3366
3367
3368/**
3369 * Sets the TPR threshold in the VMCS.
3370 *
3371 * @returns VBox status code.
3372 * @param pVCpu The cross context virtual CPU structure.
3373 * @param u32TprThreshold The TPR threshold (task-priority class only).
3374 */
3375DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3376{
3377 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3378 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
3379 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3380}
3381
3382
3383/**
3384 * Loads the guest APIC and related state.
3385 *
3386 * @returns VBox status code.
3387 * @param pVCpu The cross context virtual CPU structure.
3388 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3389 * out-of-sync. Make sure to update the required fields
3390 * before using them.
3391 */
3392DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3393{
3394 NOREF(pMixedCtx);
3395
3396 int rc = VINF_SUCCESS;
3397 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3398 {
3399 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3400 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3401 {
3402 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3403
3404 bool fPendingIntr = false;
3405 uint8_t u8Tpr = 0;
3406 uint8_t u8PendingIntr = 0;
3407 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3408 AssertRCReturn(rc, rc);
3409
3410 /*
3411 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3412 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3413 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3414 * the interrupt when we VM-exit for other reasons.
3415 */
3416 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3417 uint32_t u32TprThreshold = 0;
3418 if (fPendingIntr)
3419 {
3420 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3421 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3422 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3423 if (u8PendingPriority <= u8TprPriority)
3424 u32TprThreshold = u8PendingPriority;
3425 else
3426 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3427 }
3428
3429 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3430 AssertRCReturn(rc, rc);
3431 }
3432
3433 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3434 }
3435 return rc;
3436}
3437
3438
3439/**
3440 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3441 *
3442 * @returns Guest's interruptibility-state.
3443 * @param pVCpu The cross context virtual CPU structure.
3444 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3445 * out-of-sync. Make sure to update the required fields
3446 * before using them.
3447 *
3448 * @remarks No-long-jump zone!!!
3449 */
3450DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3451{
3452 /*
3453 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3454 */
3455 uint32_t uIntrState = 0;
3456 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3457 {
3458 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3459 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3460 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3461 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3462 {
3463 if (pMixedCtx->eflags.Bits.u1IF)
3464 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3465 else
3466 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3467 }
3468 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3469 {
3470 /*
3471 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3472 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3473 */
3474 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3475 }
3476 }
3477
3478 /*
3479 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3480 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3481 * setting this would block host-NMIs and IRET will not clear the blocking.
3482 *
3483 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3484 */
3485 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3486 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3487 {
3488 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3489 }
3490
3491 return uIntrState;
3492}
3493
3494
3495/**
3496 * Loads the guest's interruptibility-state into the guest-state area in the
3497 * VMCS.
3498 *
3499 * @returns VBox status code.
3500 * @param pVCpu The cross context virtual CPU structure.
3501 * @param uIntrState The interruptibility-state to set.
3502 */
3503static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3504{
3505 NOREF(pVCpu);
3506 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3507 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3508 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3509 AssertRC(rc);
3510 return rc;
3511}
3512
3513
3514/**
3515 * Loads the exception intercepts required for guest execution in the VMCS.
3516 *
3517 * @returns VBox status code.
3518 * @param pVCpu The cross context virtual CPU structure.
3519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3520 * out-of-sync. Make sure to update the required fields
3521 * before using them.
3522 */
3523static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3524{
3525 NOREF(pMixedCtx);
3526 int rc = VINF_SUCCESS;
3527 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
3528 {
3529 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3530 if (pVCpu->hm.s.fGIMTrapXcptUD)
3531 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3532#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3533 else
3534 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3535#endif
3536
3537 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3538 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3539
3540 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3541 AssertRCReturn(rc, rc);
3542
3543 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3544 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3545 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3546 }
3547 return rc;
3548}
3549
3550
3551/**
3552 * Loads the guest's RIP into the guest-state area in the VMCS.
3553 *
3554 * @returns VBox status code.
3555 * @param pVCpu The cross context virtual CPU structure.
3556 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3557 * out-of-sync. Make sure to update the required fields
3558 * before using them.
3559 *
3560 * @remarks No-long-jump zone!!!
3561 */
3562static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3563{
3564 int rc = VINF_SUCCESS;
3565 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3566 {
3567 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3568 AssertRCReturn(rc, rc);
3569
3570 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3571 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3572 HMCPU_CF_VALUE(pVCpu)));
3573 }
3574 return rc;
3575}
3576
3577
3578/**
3579 * Loads the guest's RSP into the guest-state area in the VMCS.
3580 *
3581 * @returns VBox status code.
3582 * @param pVCpu The cross context virtual CPU structure.
3583 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3584 * out-of-sync. Make sure to update the required fields
3585 * before using them.
3586 *
3587 * @remarks No-long-jump zone!!!
3588 */
3589static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3590{
3591 int rc = VINF_SUCCESS;
3592 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3593 {
3594 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3595 AssertRCReturn(rc, rc);
3596
3597 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3598 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3599 }
3600 return rc;
3601}
3602
3603
3604/**
3605 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3606 *
3607 * @returns VBox status code.
3608 * @param pVCpu The cross context virtual CPU structure.
3609 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3610 * out-of-sync. Make sure to update the required fields
3611 * before using them.
3612 *
3613 * @remarks No-long-jump zone!!!
3614 */
3615static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3616{
3617 int rc = VINF_SUCCESS;
3618 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3619 {
3620 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621 Let us assert it as such and use 32-bit VMWRITE. */
3622 Assert(!(pMixedCtx->rflags.u64 >> 32));
3623 X86EFLAGS Eflags = pMixedCtx->eflags;
3624 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3625 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3626 * These will never be cleared/set, unless some other part of the VMM
3627 * code is buggy - in which case we're better of finding and fixing
3628 * those bugs than hiding them. */
3629 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3630 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3631 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3632 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3633
3634 /*
3635 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3636 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3637 */
3638 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3639 {
3640 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3641 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3642 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3643 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3644 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3645 }
3646
3647 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3648 AssertRCReturn(rc, rc);
3649
3650 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3651 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3652 }
3653 return rc;
3654}
3655
3656
3657/**
3658 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3659 *
3660 * @returns VBox status code.
3661 * @param pVCpu The cross context virtual CPU structure.
3662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3663 * out-of-sync. Make sure to update the required fields
3664 * before using them.
3665 *
3666 * @remarks No-long-jump zone!!!
3667 */
3668DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3669{
3670 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3671 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3672 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3673 AssertRCReturn(rc, rc);
3674 return rc;
3675}
3676
3677
3678/**
3679 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3680 * CR0 is partially shared with the host and we have to consider the FPU bits.
3681 *
3682 * @returns VBox status code.
3683 * @param pVCpu The cross context virtual CPU structure.
3684 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685 * out-of-sync. Make sure to update the required fields
3686 * before using them.
3687 *
3688 * @remarks No-long-jump zone!!!
3689 */
3690static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3691{
3692 /*
3693 * Guest CR0.
3694 * Guest FPU.
3695 */
3696 int rc = VINF_SUCCESS;
3697 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3698 {
3699 Assert(!(pMixedCtx->cr0 >> 32));
3700 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3701 PVM pVM = pVCpu->CTX_SUFF(pVM);
3702
3703 /* The guest's view (read access) of its CR0 is unblemished. */
3704 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3705 AssertRCReturn(rc, rc);
3706 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3707
3708 /* Setup VT-x's view of the guest CR0. */
3709 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3710 if (pVM->hm.s.fNestedPaging)
3711 {
3712 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3713 {
3714 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3715 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3716 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3717 }
3718 else
3719 {
3720 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3721 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3723 }
3724
3725 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3726 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3727 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3728
3729 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3730 AssertRCReturn(rc, rc);
3731 }
3732 else
3733 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3734
3735 /*
3736 * Guest FPU bits.
3737 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3738 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3739 */
3740 u32GuestCR0 |= X86_CR0_NE;
3741 bool fInterceptNM = false;
3742 if (CPUMIsGuestFPUStateActive(pVCpu))
3743 {
3744 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3745 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3746 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3747 }
3748 else
3749 {
3750 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3751 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3752 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3753 }
3754
3755 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3756 bool fInterceptMF = false;
3757 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3758 fInterceptMF = true;
3759
3760 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3761 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3762 {
3763 Assert(PDMVmmDevHeapIsEnabled(pVM));
3764 Assert(pVM->hm.s.vmx.pRealModeTSS);
3765 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3766 fInterceptNM = true;
3767 fInterceptMF = true;
3768 }
3769 else
3770 {
3771 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3772 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3773 }
3774 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
3775
3776 if (fInterceptNM)
3777 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3778 else
3779 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3780
3781 if (fInterceptMF)
3782 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3783 else
3784 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3785
3786 /* Additional intercepts for debugging, define these yourself explicitly. */
3787#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3788 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3789 | RT_BIT(X86_XCPT_BP)
3790 | RT_BIT(X86_XCPT_DE)
3791 | RT_BIT(X86_XCPT_NM)
3792 | RT_BIT(X86_XCPT_TS)
3793 | RT_BIT(X86_XCPT_UD)
3794 | RT_BIT(X86_XCPT_NP)
3795 | RT_BIT(X86_XCPT_SS)
3796 | RT_BIT(X86_XCPT_GP)
3797 | RT_BIT(X86_XCPT_PF)
3798 | RT_BIT(X86_XCPT_MF)
3799 ;
3800#elif defined(HMVMX_ALWAYS_TRAP_PF)
3801 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3802#endif
3803
3804 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3805
3806 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3807 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3808 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3809 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3810 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3811 else
3812 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3813
3814 u32GuestCR0 |= uSetCR0;
3815 u32GuestCR0 &= uZapCR0;
3816 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3817
3818 /* Write VT-x's view of the guest CR0 into the VMCS. */
3819 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3820 AssertRCReturn(rc, rc);
3821 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3822 uZapCR0));
3823
3824 /*
3825 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3826 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3827 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3828 */
3829 uint32_t u32CR0Mask = 0;
3830 u32CR0Mask = X86_CR0_PE
3831 | X86_CR0_NE
3832 | X86_CR0_WP
3833 | X86_CR0_PG
3834 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3835 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3836 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3837
3838 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3839 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3840 * and @bugref{6944}. */
3841#if 0
3842 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3843 u32CR0Mask &= ~X86_CR0_PE;
3844#endif
3845 if (pVM->hm.s.fNestedPaging)
3846 u32CR0Mask &= ~X86_CR0_WP;
3847
3848 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3849 if (fInterceptNM)
3850 {
3851 u32CR0Mask |= X86_CR0_TS
3852 | X86_CR0_MP;
3853 }
3854
3855 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3856 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3857 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3858 AssertRCReturn(rc, rc);
3859 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3860
3861 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3862 }
3863 return rc;
3864}
3865
3866
3867/**
3868 * Loads the guest control registers (CR3, CR4) into the guest-state area
3869 * in the VMCS.
3870 *
3871 * @returns VBox strict status code.
3872 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3873 * without unrestricted guest access and the VMMDev is not presently
3874 * mapped (e.g. EFI32).
3875 *
3876 * @param pVCpu The cross context virtual CPU structure.
3877 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3878 * out-of-sync. Make sure to update the required fields
3879 * before using them.
3880 *
3881 * @remarks No-long-jump zone!!!
3882 */
3883static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3884{
3885 int rc = VINF_SUCCESS;
3886 PVM pVM = pVCpu->CTX_SUFF(pVM);
3887
3888 /*
3889 * Guest CR2.
3890 * It's always loaded in the assembler code. Nothing to do here.
3891 */
3892
3893 /*
3894 * Guest CR3.
3895 */
3896 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3897 {
3898 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3899 if (pVM->hm.s.fNestedPaging)
3900 {
3901 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3902
3903 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3904 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3905 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3906 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3907
3908 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3909 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3910 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3911
3912 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3913 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3914 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3915 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3916 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3917 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3918 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3919
3920 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3921 AssertRCReturn(rc, rc);
3922 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3923
3924 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3925 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3926 {
3927 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3928 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3929 {
3930 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3931 AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3933 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3934 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3935 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3936 AssertRCReturn(rc, rc);
3937 }
3938
3939 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3940 have Unrestricted Execution to handle the guest when it's not using paging. */
3941 GCPhysGuestCR3 = pMixedCtx->cr3;
3942 }
3943 else
3944 {
3945 /*
3946 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3947 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3948 * EPT takes care of translating it to host-physical addresses.
3949 */
3950 RTGCPHYS GCPhys;
3951 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3952
3953 /* We obtain it here every time as the guest could have relocated this PCI region. */
3954 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3955 if (RT_SUCCESS(rc))
3956 { /* likely */ }
3957 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3958 {
3959 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
3960 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3961 }
3962 else
3963 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3964
3965 GCPhysGuestCR3 = GCPhys;
3966 }
3967
3968 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3969 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3970 }
3971 else
3972 {
3973 /* Non-nested paging case, just use the hypervisor's CR3. */
3974 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3975
3976 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3977 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3978 }
3979 AssertRCReturn(rc, rc);
3980
3981 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3982 }
3983
3984 /*
3985 * Guest CR4.
3986 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3987 */
3988 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3989 {
3990 Assert(!(pMixedCtx->cr4 >> 32));
3991 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3992
3993 /* The guest's view of its CR4 is unblemished. */
3994 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3995 AssertRCReturn(rc, rc);
3996 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3997
3998 /* Setup VT-x's view of the guest CR4. */
3999 /*
4000 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4001 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4002 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4003 */
4004 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4005 {
4006 Assert(pVM->hm.s.vmx.pRealModeTSS);
4007 Assert(PDMVmmDevHeapIsEnabled(pVM));
4008 u32GuestCR4 &= ~X86_CR4_VME;
4009 }
4010
4011 if (pVM->hm.s.fNestedPaging)
4012 {
4013 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4014 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4015 {
4016 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4017 u32GuestCR4 |= X86_CR4_PSE;
4018 /* Our identity mapping is a 32-bit page directory. */
4019 u32GuestCR4 &= ~X86_CR4_PAE;
4020 }
4021 /* else use guest CR4.*/
4022 }
4023 else
4024 {
4025 /*
4026 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4027 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4028 */
4029 switch (pVCpu->hm.s.enmShadowMode)
4030 {
4031 case PGMMODE_REAL: /* Real-mode. */
4032 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4033 case PGMMODE_32_BIT: /* 32-bit paging. */
4034 {
4035 u32GuestCR4 &= ~X86_CR4_PAE;
4036 break;
4037 }
4038
4039 case PGMMODE_PAE: /* PAE paging. */
4040 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4041 {
4042 u32GuestCR4 |= X86_CR4_PAE;
4043 break;
4044 }
4045
4046 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4047 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4048#ifdef VBOX_ENABLE_64_BITS_GUESTS
4049 break;
4050#endif
4051 default:
4052 AssertFailed();
4053 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4054 }
4055 }
4056
4057 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4058 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4059 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4060 u32GuestCR4 |= uSetCR4;
4061 u32GuestCR4 &= uZapCR4;
4062
4063 /* Write VT-x's view of the guest CR4 into the VMCS. */
4064 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4065 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4066 AssertRCReturn(rc, rc);
4067
4068 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4069 uint32_t u32CR4Mask = X86_CR4_VME
4070 | X86_CR4_PAE
4071 | X86_CR4_PGE
4072 | X86_CR4_PSE
4073 | X86_CR4_VMXE;
4074 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4075 u32CR4Mask |= X86_CR4_OSXSAVE;
4076 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4077 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4078 AssertRCReturn(rc, rc);
4079
4080 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4081 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4082
4083 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4084 }
4085 return rc;
4086}
4087
4088
4089/**
4090 * Loads the guest debug registers into the guest-state area in the VMCS.
4091 *
4092 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4093 *
4094 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4095 *
4096 * @returns VBox status code.
4097 * @param pVCpu The cross context virtual CPU structure.
4098 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4099 * out-of-sync. Make sure to update the required fields
4100 * before using them.
4101 *
4102 * @remarks No-long-jump zone!!!
4103 */
4104static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4105{
4106 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4107 return VINF_SUCCESS;
4108
4109#ifdef VBOX_STRICT
4110 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4111 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4112 {
4113 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4114 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4115 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4116 }
4117#endif
4118
4119 int rc;
4120 PVM pVM = pVCpu->CTX_SUFF(pVM);
4121 bool fSteppingDB = false;
4122 bool fInterceptMovDRx = false;
4123 if (pVCpu->hm.s.fSingleInstruction)
4124 {
4125 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4126 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4127 {
4128 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4129 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4130 AssertRCReturn(rc, rc);
4131 Assert(fSteppingDB == false);
4132 }
4133 else
4134 {
4135 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4136 pVCpu->hm.s.fClearTrapFlag = true;
4137 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4138 fSteppingDB = true;
4139 }
4140 }
4141
4142 if ( fSteppingDB
4143 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4144 {
4145 /*
4146 * Use the combined guest and host DRx values found in the hypervisor
4147 * register set because the debugger has breakpoints active or someone
4148 * is single stepping on the host side without a monitor trap flag.
4149 *
4150 * Note! DBGF expects a clean DR6 state before executing guest code.
4151 */
4152#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4153 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4154 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4155 {
4156 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4157 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4158 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4159 }
4160 else
4161#endif
4162 if (!CPUMIsHyperDebugStateActive(pVCpu))
4163 {
4164 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4165 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4166 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4167 }
4168
4169 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4170 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4171 AssertRCReturn(rc, rc);
4172
4173 pVCpu->hm.s.fUsingHyperDR7 = true;
4174 fInterceptMovDRx = true;
4175 }
4176 else
4177 {
4178 /*
4179 * If the guest has enabled debug registers, we need to load them prior to
4180 * executing guest code so they'll trigger at the right time.
4181 */
4182 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4183 {
4184#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4185 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4186 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4187 {
4188 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4189 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4190 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4191 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4192 }
4193 else
4194#endif
4195 if (!CPUMIsGuestDebugStateActive(pVCpu))
4196 {
4197 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4198 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4199 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4200 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4201 }
4202 Assert(!fInterceptMovDRx);
4203 }
4204 /*
4205 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4206 * must intercept #DB in order to maintain a correct DR6 guest value, and
4207 * because we need to intercept it to prevent nested #DBs from hanging the
4208 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4209 */
4210#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4211 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4212 && !CPUMIsGuestDebugStateActive(pVCpu))
4213#else
4214 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4215#endif
4216 {
4217 fInterceptMovDRx = true;
4218 }
4219
4220 /* Update guest DR7. */
4221 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4222 AssertRCReturn(rc, rc);
4223
4224 pVCpu->hm.s.fUsingHyperDR7 = false;
4225 }
4226
4227 /*
4228 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4229 */
4230 if (fInterceptMovDRx)
4231 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 else
4233 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4235 AssertRCReturn(rc, rc);
4236
4237 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4238 return VINF_SUCCESS;
4239}
4240
4241
4242#ifdef VBOX_STRICT
4243/**
4244 * Strict function to validate segment registers.
4245 *
4246 * @remarks ASSUMES CR0 is up to date.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4249{
4250 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4251 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4252 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4254 && ( !CPUMIsGuestInRealModeEx(pCtx)
4255 && !CPUMIsGuestInV86ModeEx(pCtx)))
4256 {
4257 /* Protected mode checks */
4258 /* CS */
4259 Assert(pCtx->cs.Attr.n.u1Present);
4260 Assert(!(pCtx->cs.Attr.u & 0xf00));
4261 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->cs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4265 || (pCtx->cs.Attr.n.u1Granularity));
4266 /* CS cannot be loaded with NULL in protected mode. */
4267 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4268 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4269 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4270 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4271 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4272 else
4273 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4274 /* SS */
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4277 if ( !(pCtx->cr0 & X86_CR0_PE)
4278 || pCtx->cs.Attr.n.u4Type == 3)
4279 {
4280 Assert(!pCtx->ss.Attr.n.u2Dpl);
4281 }
4282 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4286 Assert(pCtx->ss.Attr.n.u1Present);
4287 Assert(!(pCtx->ss.Attr.u & 0xf00));
4288 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ss.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4292 || (pCtx->ss.Attr.n.u1Granularity));
4293 }
4294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4295 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->ds.Attr.n.u1Present);
4299 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->ds.Attr.u & 0xf00));
4301 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4305 || (pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->es.Attr.n.u1Present);
4313 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->es.Attr.u & 0xf00));
4315 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4319 || (pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->fs.Attr.n.u1Present);
4327 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->fs.Attr.u & 0xf00));
4329 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4333 || (pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->gs.Attr.n.u1Present);
4341 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->gs.Attr.u & 0xf00));
4343 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4347 || (pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 /* 64-bit capable CPUs. */
4352# if HC_ARCH_BITS == 64
4353 Assert(!(pCtx->cs.u64Base >> 32));
4354 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4355 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4356 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4357# endif
4358 }
4359 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4360 || ( CPUMIsGuestInRealModeEx(pCtx)
4361 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4362 {
4363 /* Real and v86 mode checks. */
4364 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4365 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4366 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4367 {
4368 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4369 }
4370 else
4371 {
4372 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4373 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4374 }
4375
4376 /* CS */
4377 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4378 Assert(pCtx->cs.u32Limit == 0xffff);
4379 Assert(u32CSAttr == 0xf3);
4380 /* SS */
4381 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4382 Assert(pCtx->ss.u32Limit == 0xffff);
4383 Assert(u32SSAttr == 0xf3);
4384 /* DS */
4385 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4386 Assert(pCtx->ds.u32Limit == 0xffff);
4387 Assert(u32DSAttr == 0xf3);
4388 /* ES */
4389 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4390 Assert(pCtx->es.u32Limit == 0xffff);
4391 Assert(u32ESAttr == 0xf3);
4392 /* FS */
4393 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4394 Assert(pCtx->fs.u32Limit == 0xffff);
4395 Assert(u32FSAttr == 0xf3);
4396 /* GS */
4397 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4398 Assert(pCtx->gs.u32Limit == 0xffff);
4399 Assert(u32GSAttr == 0xf3);
4400 /* 64-bit capable CPUs. */
4401# if HC_ARCH_BITS == 64
4402 Assert(!(pCtx->cs.u64Base >> 32));
4403 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4404 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4405 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4406# endif
4407 }
4408}
4409#endif /* VBOX_STRICT */
4410
4411
4412/**
4413 * Writes a guest segment register into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVCpu The cross context virtual CPU structure.
4417 * @param idxSel Index of the selector in the VMCS.
4418 * @param idxLimit Index of the segment limit in the VMCS.
4419 * @param idxBase Index of the segment base in the VMCS.
4420 * @param idxAccess Index of the access rights of the segment in the VMCS.
4421 * @param pSelReg Pointer to the segment selector.
4422 *
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4426 uint32_t idxAccess, PCPUMSELREG pSelReg)
4427{
4428 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4429 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4430 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4431 AssertRCReturn(rc, rc);
4432
4433 uint32_t u32Access = pSelReg->Attr.u;
4434 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4435 {
4436 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4437 u32Access = 0xf3;
4438 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4439 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4440 }
4441 else
4442 {
4443 /*
4444 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4445 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4446 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4447 * loaded in protected-mode have their attribute as 0.
4448 */
4449 if (!u32Access)
4450 u32Access = X86DESCATTR_UNUSABLE;
4451 }
4452
4453 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4454 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4455 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4456
4457 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4458 AssertRCReturn(rc, rc);
4459 return rc;
4460}
4461
4462
4463/**
4464 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4465 * into the guest-state area in the VMCS.
4466 *
4467 * @returns VBox status code.
4468 * @param pVCpu The cross context virtual CPU structure.
4469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4470 * out-of-sync. Make sure to update the required fields
4471 * before using them.
4472 *
4473 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4474 * @remarks No-long-jump zone!!!
4475 */
4476static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4477{
4478 int rc = VERR_INTERNAL_ERROR_5;
4479 PVM pVM = pVCpu->CTX_SUFF(pVM);
4480
4481 /*
4482 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4483 */
4484 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4485 {
4486 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4487 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4488 {
4489 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4490 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4491 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4492 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4493 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4494 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4495 }
4496
4497#ifdef VBOX_WITH_REM
4498 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4499 {
4500 Assert(pVM->hm.s.vmx.pRealModeTSS);
4501 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4502 if ( pVCpu->hm.s.vmx.fWasInRealMode
4503 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4504 {
4505 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4506 in real-mode (e.g. OpenBSD 4.0) */
4507 REMFlushTBs(pVM);
4508 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4509 pVCpu->hm.s.vmx.fWasInRealMode = false;
4510 }
4511 }
4512#endif
4513 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4514 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4515 AssertRCReturn(rc, rc);
4516 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4517 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4518 AssertRCReturn(rc, rc);
4519 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4520 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4521 AssertRCReturn(rc, rc);
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4523 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4526 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4529 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4530 AssertRCReturn(rc, rc);
4531
4532#ifdef VBOX_STRICT
4533 /* Validate. */
4534 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4535#endif
4536
4537 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4538 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4539 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4540 }
4541
4542 /*
4543 * Guest TR.
4544 */
4545 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4546 {
4547 /*
4548 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4549 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4550 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4551 */
4552 uint16_t u16Sel = 0;
4553 uint32_t u32Limit = 0;
4554 uint64_t u64Base = 0;
4555 uint32_t u32AccessRights = 0;
4556
4557 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4558 {
4559 u16Sel = pMixedCtx->tr.Sel;
4560 u32Limit = pMixedCtx->tr.u32Limit;
4561 u64Base = pMixedCtx->tr.u64Base;
4562 u32AccessRights = pMixedCtx->tr.Attr.u;
4563 }
4564 else
4565 {
4566 Assert(pVM->hm.s.vmx.pRealModeTSS);
4567 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4568
4569 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4570 RTGCPHYS GCPhys;
4571 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4572 AssertRCReturn(rc, rc);
4573
4574 X86DESCATTR DescAttr;
4575 DescAttr.u = 0;
4576 DescAttr.n.u1Present = 1;
4577 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4578
4579 u16Sel = 0;
4580 u32Limit = HM_VTX_TSS_SIZE;
4581 u64Base = GCPhys; /* in real-mode phys = virt. */
4582 u32AccessRights = DescAttr.u;
4583 }
4584
4585 /* Validate. */
4586 Assert(!(u16Sel & RT_BIT(2)));
4587 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4588 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4589 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4590 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4591 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4592 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4593 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4594 Assert( (u32Limit & 0xfff) == 0xfff
4595 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4596 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4597 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4598
4599 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4600 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4601 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4603 AssertRCReturn(rc, rc);
4604
4605 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4606 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4607 }
4608
4609 /*
4610 * Guest GDTR.
4611 */
4612 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4613 {
4614 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4615 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4616 AssertRCReturn(rc, rc);
4617
4618 /* Validate. */
4619 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4620
4621 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4622 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4623 }
4624
4625 /*
4626 * Guest LDTR.
4627 */
4628 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4629 {
4630 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4631 uint32_t u32Access = 0;
4632 if (!pMixedCtx->ldtr.Attr.u)
4633 u32Access = X86DESCATTR_UNUSABLE;
4634 else
4635 u32Access = pMixedCtx->ldtr.Attr.u;
4636
4637 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4638 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4639 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4640 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4641 AssertRCReturn(rc, rc);
4642
4643 /* Validate. */
4644 if (!(u32Access & X86DESCATTR_UNUSABLE))
4645 {
4646 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4647 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4648 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4649 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4650 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4651 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4652 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4653 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4654 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4655 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4656 }
4657
4658 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4659 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4660 }
4661
4662 /*
4663 * Guest IDTR.
4664 */
4665 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4666 {
4667 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4668 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4669 AssertRCReturn(rc, rc);
4670
4671 /* Validate. */
4672 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4673
4674 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4675 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4676 }
4677
4678 return VINF_SUCCESS;
4679}
4680
4681
4682/**
4683 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4684 * areas.
4685 *
4686 * These MSRs will automatically be loaded to the host CPU on every successful
4687 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4688 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4689 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4690 *
4691 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4692 *
4693 * @returns VBox status code.
4694 * @param pVCpu The cross context virtual CPU structure.
4695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4696 * out-of-sync. Make sure to update the required fields
4697 * before using them.
4698 *
4699 * @remarks No-long-jump zone!!!
4700 */
4701static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4702{
4703 AssertPtr(pVCpu);
4704 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4705
4706 /*
4707 * MSRs that we use the auto-load/store MSR area in the VMCS.
4708 */
4709 PVM pVM = pVCpu->CTX_SUFF(pVM);
4710 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4711 {
4712 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4713#if HC_ARCH_BITS == 32
4714 if (pVM->hm.s.fAllow64BitGuests)
4715 {
4716 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4717 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4718 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4719 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4720 AssertRCReturn(rc, rc);
4721# ifdef LOG_ENABLED
4722 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4723 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4724 {
4725 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4726 pMsr->u64Value));
4727 }
4728# endif
4729 }
4730#endif
4731 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4732 }
4733
4734 /*
4735 * Guest Sysenter MSRs.
4736 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4737 * VM-exits on WRMSRs for these MSRs.
4738 */
4739 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4740 {
4741 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4742 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4743 }
4744
4745 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4746 {
4747 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4748 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4749 }
4750
4751 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4752 {
4753 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4754 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4755 }
4756
4757 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4758 {
4759 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4760 {
4761 /*
4762 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4763 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4764 */
4765 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4766 {
4767 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4768 AssertRCReturn(rc,rc);
4769 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4770 }
4771 else
4772 {
4773 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4774 NULL /* pfAddedAndUpdated */);
4775 AssertRCReturn(rc, rc);
4776
4777 /* We need to intercept reads too, see @bugref{7386#c16}. */
4778 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4779 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4780 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4781 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4782 }
4783 }
4784 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4787 }
4788
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Loads the guest activity state into the guest-state area in the VMCS.
4795 *
4796 * @returns VBox status code.
4797 * @param pVCpu The cross context virtual CPU structure.
4798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799 * out-of-sync. Make sure to update the required fields
4800 * before using them.
4801 *
4802 * @remarks No-long-jump zone!!!
4803 */
4804static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4805{
4806 NOREF(pMixedCtx);
4807 /** @todo See if we can make use of other states, e.g.
4808 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4810 {
4811 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4812 AssertRCReturn(rc, rc);
4813
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4815 }
4816 return VINF_SUCCESS;
4817}
4818
4819
4820/**
4821 * Sets up the appropriate function to run guest code.
4822 *
4823 * @returns VBox status code.
4824 * @param pVCpu The cross context virtual CPU structure.
4825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4826 * out-of-sync. Make sure to update the required fields
4827 * before using them.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4832{
4833 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4834 {
4835#ifndef VBOX_ENABLE_64_BITS_GUESTS
4836 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4837#endif
4838 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4839#if HC_ARCH_BITS == 32
4840 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4841 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4842 {
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4847 | HM_CHANGED_VMX_ENTRY_CTLS
4848 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4849 }
4850 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4851
4852 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4853 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4854 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4855 }
4856#else
4857 /* 64-bit host. */
4858 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4859#endif
4860 }
4861 else
4862 {
4863 /* Guest is not in long mode, use the 32-bit handler. */
4864#if HC_ARCH_BITS == 32
4865 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4866 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4867 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4868 {
4869 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4870 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4871 | HM_CHANGED_VMX_ENTRY_CTLS
4872 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4873 }
4874# ifdef VBOX_ENABLE_64_BITS_GUESTS
4875 /* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design. See @bugref{8432#c7}. */
4876 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4877 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4878 else
4879 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4880# else
4881 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4882# endif
4883#else
4884 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4885#endif
4886 }
4887 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4888 return VINF_SUCCESS;
4889}
4890
4891
4892/**
4893 * Wrapper for running the guest code in VT-x.
4894 *
4895 * @returns VBox status code, no informational status codes.
4896 * @param pVM The cross context VM structure.
4897 * @param pVCpu The cross context virtual CPU structure.
4898 * @param pCtx Pointer to the guest-CPU context.
4899 *
4900 * @remarks No-long-jump zone!!!
4901 */
4902DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4903{
4904 /*
4905 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4906 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4907 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4908 */
4909 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4910 /** @todo Add stats for resume vs launch. */
4911#ifdef VBOX_WITH_KERNEL_USING_XMM
4912 int rc = HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4913#else
4914 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4915#endif
4916 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4917 return rc;
4918}
4919
4920
4921/**
4922 * Reports world-switch error and dumps some useful debug info.
4923 *
4924 * @param pVM The cross context VM structure.
4925 * @param pVCpu The cross context virtual CPU structure.
4926 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4927 * @param pCtx Pointer to the guest-CPU context.
4928 * @param pVmxTransient Pointer to the VMX transient structure (only
4929 * exitReason updated).
4930 */
4931static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4932{
4933 Assert(pVM);
4934 Assert(pVCpu);
4935 Assert(pCtx);
4936 Assert(pVmxTransient);
4937 HMVMX_ASSERT_PREEMPT_SAFE();
4938
4939 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4940 switch (rcVMRun)
4941 {
4942 case VERR_VMX_INVALID_VMXON_PTR:
4943 AssertFailed();
4944 break;
4945 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4946 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4947 {
4948 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4949 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4950 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4951 AssertRC(rc);
4952
4953 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4954 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4955 Cannot do it here as we may have been long preempted. */
4956
4957#ifdef VBOX_STRICT
4958 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4959 pVmxTransient->uExitReason));
4960 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4961 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4962 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4963 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4964 else
4965 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4966 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4967 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4968
4969 /* VMX control bits. */
4970 uint32_t u32Val;
4971 uint64_t u64Val;
4972 RTHCUINTREG uHCReg;
4973 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4974 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4975 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4976 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4979 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4980 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4981 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4982 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4983 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4984 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4985 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4986 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4988 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4989 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4990 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4991 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4992 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4993 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4994 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5005 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5006 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5007 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5008 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5009 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5010 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5011 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5012 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5013 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5014 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5015
5016 /* Guest bits. */
5017 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5018 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5019 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5020 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5021 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5022 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5023 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5024 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5025
5026 /* Host bits. */
5027 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5028 Log4(("Host CR0 %#RHr\n", uHCReg));
5029 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5030 Log4(("Host CR3 %#RHr\n", uHCReg));
5031 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5032 Log4(("Host CR4 %#RHr\n", uHCReg));
5033
5034 RTGDTR HostGdtr;
5035 PCX86DESCHC pDesc;
5036 ASMGetGDTR(&HostGdtr);
5037 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5038 Log4(("Host CS %#08x\n", u32Val));
5039 if (u32Val < HostGdtr.cbGdt)
5040 {
5041 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5042 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5043 }
5044
5045 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5046 Log4(("Host DS %#08x\n", u32Val));
5047 if (u32Val < HostGdtr.cbGdt)
5048 {
5049 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5050 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5051 }
5052
5053 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5054 Log4(("Host ES %#08x\n", u32Val));
5055 if (u32Val < HostGdtr.cbGdt)
5056 {
5057 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5058 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5059 }
5060
5061 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5062 Log4(("Host FS %#08x\n", u32Val));
5063 if (u32Val < HostGdtr.cbGdt)
5064 {
5065 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5066 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5067 }
5068
5069 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5070 Log4(("Host GS %#08x\n", u32Val));
5071 if (u32Val < HostGdtr.cbGdt)
5072 {
5073 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5074 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5075 }
5076
5077 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5078 Log4(("Host SS %#08x\n", u32Val));
5079 if (u32Val < HostGdtr.cbGdt)
5080 {
5081 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5082 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5083 }
5084
5085 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5086 Log4(("Host TR %#08x\n", u32Val));
5087 if (u32Val < HostGdtr.cbGdt)
5088 {
5089 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5090 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5091 }
5092
5093 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5094 Log4(("Host TR Base %#RHv\n", uHCReg));
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5096 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5098 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5099 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5100 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5101 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5102 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5103 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5104 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5105 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5106 Log4(("Host RSP %#RHv\n", uHCReg));
5107 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5108 Log4(("Host RIP %#RHv\n", uHCReg));
5109# if HC_ARCH_BITS == 64
5110 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5111 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5112 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5113 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5114 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5115 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5116# endif
5117#endif /* VBOX_STRICT */
5118 break;
5119 }
5120
5121 default:
5122 /* Impossible */
5123 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5124 break;
5125 }
5126 NOREF(pVM); NOREF(pCtx);
5127}
5128
5129
5130#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5131#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5132# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5133#endif
5134#ifdef VBOX_STRICT
5135static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5136{
5137 switch (idxField)
5138 {
5139 case VMX_VMCS_GUEST_RIP:
5140 case VMX_VMCS_GUEST_RSP:
5141 case VMX_VMCS_GUEST_SYSENTER_EIP:
5142 case VMX_VMCS_GUEST_SYSENTER_ESP:
5143 case VMX_VMCS_GUEST_GDTR_BASE:
5144 case VMX_VMCS_GUEST_IDTR_BASE:
5145 case VMX_VMCS_GUEST_CS_BASE:
5146 case VMX_VMCS_GUEST_DS_BASE:
5147 case VMX_VMCS_GUEST_ES_BASE:
5148 case VMX_VMCS_GUEST_FS_BASE:
5149 case VMX_VMCS_GUEST_GS_BASE:
5150 case VMX_VMCS_GUEST_SS_BASE:
5151 case VMX_VMCS_GUEST_LDTR_BASE:
5152 case VMX_VMCS_GUEST_TR_BASE:
5153 case VMX_VMCS_GUEST_CR3:
5154 return true;
5155 }
5156 return false;
5157}
5158
5159static bool hmR0VmxIsValidReadField(uint32_t idxField)
5160{
5161 switch (idxField)
5162 {
5163 /* Read-only fields. */
5164 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5165 return true;
5166 }
5167 /* Remaining readable fields should also be writable. */
5168 return hmR0VmxIsValidWriteField(idxField);
5169}
5170#endif /* VBOX_STRICT */
5171
5172
5173/**
5174 * Executes the specified handler in 64-bit mode.
5175 *
5176 * @returns VBox status code (no informational status codes).
5177 * @param pVM The cross context VM structure.
5178 * @param pVCpu The cross context virtual CPU structure.
5179 * @param pCtx Pointer to the guest CPU context.
5180 * @param enmOp The operation to perform.
5181 * @param cParams Number of parameters.
5182 * @param paParam Array of 32-bit parameters.
5183 */
5184VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5185 uint32_t cParams, uint32_t *paParam)
5186{
5187 NOREF(pCtx);
5188
5189 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5190 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5191 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5192 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5193
5194#ifdef VBOX_STRICT
5195 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5196 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5197
5198 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5199 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5200#endif
5201
5202 /* Disable interrupts. */
5203 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5204
5205#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5206 RTCPUID idHostCpu = RTMpCpuId();
5207 CPUMR0SetLApic(pVCpu, idHostCpu);
5208#endif
5209
5210 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5211 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5212
5213 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5214 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5215
5216 /* Leave VMX Root Mode. */
5217 VMXDisable();
5218
5219 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5220
5221 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5222 CPUMSetHyperEIP(pVCpu, enmOp);
5223 for (int i = (int)cParams - 1; i >= 0; i--)
5224 CPUMPushHyper(pVCpu, paParam[i]);
5225
5226 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5227
5228 /* Call the switcher. */
5229 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5230 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5231
5232 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5233 /* Make sure the VMX instructions don't cause #UD faults. */
5234 SUPR0ChangeCR4(X86_CR4_VMXE, ~0);
5235
5236 /* Re-enter VMX Root Mode */
5237 int rc2 = VMXEnable(HCPhysCpuPage);
5238 if (RT_FAILURE(rc2))
5239 {
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241 ASMSetFlags(fOldEFlags);
5242 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5243 return rc2;
5244 }
5245
5246 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5247 AssertRC(rc2);
5248 Assert(!(ASMGetFlags() & X86_EFL_IF));
5249 ASMSetFlags(fOldEFlags);
5250 return rc;
5251}
5252
5253
5254/**
5255 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5256 * supporting 64-bit guests.
5257 *
5258 * @returns VBox status code.
5259 * @param fResume Whether to VMLAUNCH or VMRESUME.
5260 * @param pCtx Pointer to the guest-CPU context.
5261 * @param pCache Pointer to the VMCS cache.
5262 * @param pVM The cross context VM structure.
5263 * @param pVCpu The cross context virtual CPU structure.
5264 */
5265DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5266{
5267 NOREF(fResume);
5268
5269 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
5270 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5271
5272#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5273 pCache->uPos = 1;
5274 pCache->interPD = PGMGetInterPaeCR3(pVM);
5275 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5276#endif
5277
5278#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5279 pCache->TestIn.HCPhysCpuPage = 0;
5280 pCache->TestIn.HCPhysVmcs = 0;
5281 pCache->TestIn.pCache = 0;
5282 pCache->TestOut.HCPhysVmcs = 0;
5283 pCache->TestOut.pCache = 0;
5284 pCache->TestOut.pCtx = 0;
5285 pCache->TestOut.eflags = 0;
5286#else
5287 NOREF(pCache);
5288#endif
5289
5290 uint32_t aParam[10];
5291 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5292 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5293 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5294 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5295 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5296 aParam[5] = 0;
5297 aParam[6] = VM_RC_ADDR(pVM, pVM);
5298 aParam[7] = 0;
5299 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5300 aParam[9] = 0;
5301
5302#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5303 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5304 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5305#endif
5306 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5307
5308#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5309 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5310 Assert(pCtx->dr[4] == 10);
5311 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5312#endif
5313
5314#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5315 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5316 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5317 pVCpu->hm.s.vmx.HCPhysVmcs));
5318 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5319 pCache->TestOut.HCPhysVmcs));
5320 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5321 pCache->TestOut.pCache));
5322 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5323 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5324 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5325 pCache->TestOut.pCtx));
5326 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5327#endif
5328 return rc;
5329}
5330
5331
5332/**
5333 * Initialize the VMCS-Read cache.
5334 *
5335 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5336 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5337 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5338 * (those that have a 32-bit FULL & HIGH part).
5339 *
5340 * @returns VBox status code.
5341 * @param pVM The cross context VM structure.
5342 * @param pVCpu The cross context virtual CPU structure.
5343 */
5344static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5345{
5346#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5347{ \
5348 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5349 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5350 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5351 ++cReadFields; \
5352}
5353
5354 AssertPtr(pVM);
5355 AssertPtr(pVCpu);
5356 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5357 uint32_t cReadFields = 0;
5358
5359 /*
5360 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5361 * and serve to indicate exceptions to the rules.
5362 */
5363
5364 /* Guest-natural selector base fields. */
5365#if 0
5366 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5369#endif
5370 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5371 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5374 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5377 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5378 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5379 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5382#if 0
5383 /* Unused natural width guest-state fields. */
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5386#endif
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5389
5390 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5391#if 0
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5401#endif
5402
5403 /* Natural width guest-state fields. */
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5405#if 0
5406 /* Currently unused field. */
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5408#endif
5409
5410 if (pVM->hm.s.fNestedPaging)
5411 {
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5413 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5414 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5415 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5416 }
5417 else
5418 {
5419 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5420 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5421 }
5422
5423#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5424 return VINF_SUCCESS;
5425}
5426
5427
5428/**
5429 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5430 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5431 * darwin, running 64-bit guests).
5432 *
5433 * @returns VBox status code.
5434 * @param pVCpu The cross context virtual CPU structure.
5435 * @param idxField The VMCS field encoding.
5436 * @param u64Val 16, 32 or 64-bit value.
5437 */
5438VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5439{
5440 int rc;
5441 switch (idxField)
5442 {
5443 /*
5444 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5445 */
5446 /* 64-bit Control fields. */
5447 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5448 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5449 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5450 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5451 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5452 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5453 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5454 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5455 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5456 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5457 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5458 case VMX_VMCS64_CTRL_EPTP_FULL:
5459 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5460 /* 64-bit Guest-state fields. */
5461 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5462 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5463 case VMX_VMCS64_GUEST_PAT_FULL:
5464 case VMX_VMCS64_GUEST_EFER_FULL:
5465 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5466 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5467 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5468 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5469 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5470 /* 64-bit Host-state fields. */
5471 case VMX_VMCS64_HOST_PAT_FULL:
5472 case VMX_VMCS64_HOST_EFER_FULL:
5473 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5474 {
5475 rc = VMXWriteVmcs32(idxField, u64Val);
5476 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5477 break;
5478 }
5479
5480 /*
5481 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5482 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5483 */
5484 /* Natural-width Guest-state fields. */
5485 case VMX_VMCS_GUEST_CR3:
5486 case VMX_VMCS_GUEST_ES_BASE:
5487 case VMX_VMCS_GUEST_CS_BASE:
5488 case VMX_VMCS_GUEST_SS_BASE:
5489 case VMX_VMCS_GUEST_DS_BASE:
5490 case VMX_VMCS_GUEST_FS_BASE:
5491 case VMX_VMCS_GUEST_GS_BASE:
5492 case VMX_VMCS_GUEST_LDTR_BASE:
5493 case VMX_VMCS_GUEST_TR_BASE:
5494 case VMX_VMCS_GUEST_GDTR_BASE:
5495 case VMX_VMCS_GUEST_IDTR_BASE:
5496 case VMX_VMCS_GUEST_RSP:
5497 case VMX_VMCS_GUEST_RIP:
5498 case VMX_VMCS_GUEST_SYSENTER_ESP:
5499 case VMX_VMCS_GUEST_SYSENTER_EIP:
5500 {
5501 if (!(u64Val >> 32))
5502 {
5503 /* If this field is 64-bit, VT-x will zero out the top bits. */
5504 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5505 }
5506 else
5507 {
5508 /* Assert that only the 32->64 switcher case should ever come here. */
5509 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5510 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5511 }
5512 break;
5513 }
5514
5515 default:
5516 {
5517 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5518 rc = VERR_INVALID_PARAMETER;
5519 break;
5520 }
5521 }
5522 AssertRCReturn(rc, rc);
5523 return rc;
5524}
5525
5526
5527/**
5528 * Queue up a VMWRITE by using the VMCS write cache.
5529 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5530 *
5531 * @param pVCpu The cross context virtual CPU structure.
5532 * @param idxField The VMCS field encoding.
5533 * @param u64Val 16, 32 or 64-bit value.
5534 */
5535VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5536{
5537 AssertPtr(pVCpu);
5538 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5539
5540 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5541 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5542
5543 /* Make sure there are no duplicates. */
5544 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5545 {
5546 if (pCache->Write.aField[i] == idxField)
5547 {
5548 pCache->Write.aFieldVal[i] = u64Val;
5549 return VINF_SUCCESS;
5550 }
5551 }
5552
5553 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5554 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5555 pCache->Write.cValidEntries++;
5556 return VINF_SUCCESS;
5557}
5558#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5559
5560
5561/**
5562 * Sets up the usage of TSC-offsetting and updates the VMCS.
5563 *
5564 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5565 * VMX preemption timer.
5566 *
5567 * @returns VBox status code.
5568 * @param pVM The cross context VM structure.
5569 * @param pVCpu The cross context virtual CPU structure.
5570 *
5571 * @remarks No-long-jump zone!!!
5572 */
5573static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5574{
5575 int rc;
5576 bool fOffsettedTsc;
5577 bool fParavirtTsc;
5578 if (pVM->hm.s.vmx.fUsePreemptTimer)
5579 {
5580 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5581 &fOffsettedTsc, &fParavirtTsc);
5582
5583 /* Make sure the returned values have sane upper and lower boundaries. */
5584 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5585 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5586 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5587 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5588
5589 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5590 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5591 }
5592 else
5593 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5594
5595 /** @todo later optimize this to be done elsewhere and not before every
5596 * VM-entry. */
5597 if (fParavirtTsc)
5598 {
5599 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5600 information before every VM-entry, hence disable it for performance sake. */
5601#if 0
5602 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5603 AssertRC(rc);
5604#endif
5605 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5606 }
5607
5608 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5609 {
5610 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5611 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5612
5613 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5614 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5615 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5616 }
5617 else
5618 {
5619 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5620 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5621 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5622 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5623 }
5624}
5625
5626
5627/**
5628 * Determines if an exception is a contributory exception.
5629 *
5630 * Contributory exceptions are ones which can cause double-faults unless the
5631 * original exception was a benign exception. Page-fault is intentionally not
5632 * included here as it's a conditional contributory exception.
5633 *
5634 * @returns true if the exception is contributory, false otherwise.
5635 * @param uVector The exception vector.
5636 */
5637DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5638{
5639 switch (uVector)
5640 {
5641 case X86_XCPT_GP:
5642 case X86_XCPT_SS:
5643 case X86_XCPT_NP:
5644 case X86_XCPT_TS:
5645 case X86_XCPT_DE:
5646 return true;
5647 default:
5648 break;
5649 }
5650 return false;
5651}
5652
5653
5654/**
5655 * Sets an event as a pending event to be injected into the guest.
5656 *
5657 * @param pVCpu The cross context virtual CPU structure.
5658 * @param u32IntInfo The VM-entry interruption-information field.
5659 * @param cbInstr The VM-entry instruction length in bytes (for software
5660 * interrupts, exceptions and privileged software
5661 * exceptions).
5662 * @param u32ErrCode The VM-entry exception error code.
5663 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5664 * page-fault.
5665 *
5666 * @remarks Statistics counter assumes this is a guest event being injected or
5667 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5668 * always incremented.
5669 */
5670DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5671 RTGCUINTPTR GCPtrFaultAddress)
5672{
5673 Assert(!pVCpu->hm.s.Event.fPending);
5674 pVCpu->hm.s.Event.fPending = true;
5675 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5676 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5677 pVCpu->hm.s.Event.cbInstr = cbInstr;
5678 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5679}
5680
5681
5682/**
5683 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5684 *
5685 * @param pVCpu The cross context virtual CPU structure.
5686 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5687 * out-of-sync. Make sure to update the required fields
5688 * before using them.
5689 */
5690DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5691{
5692 NOREF(pMixedCtx);
5693 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5694 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5695 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5696 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5697}
5698
5699
5700/**
5701 * Handle a condition that occurred while delivering an event through the guest
5702 * IDT.
5703 *
5704 * @returns Strict VBox status code (i.e. informational status codes too).
5705 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5706 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5707 * to continue execution of the guest which will delivery the \#DF.
5708 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5709 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5710 *
5711 * @param pVCpu The cross context virtual CPU structure.
5712 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5713 * out-of-sync. Make sure to update the required fields
5714 * before using them.
5715 * @param pVmxTransient Pointer to the VMX transient structure.
5716 *
5717 * @remarks No-long-jump zone!!!
5718 */
5719static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5720{
5721 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5722
5723 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5724 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5725
5726 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5727 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5728 {
5729 uint32_t uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5730 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5731
5732 typedef enum
5733 {
5734 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5735 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5736 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5737 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
5738 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5739 } VMXREFLECTXCPT;
5740
5741 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5742 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5743 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5744 {
5745 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5746 {
5747 enmReflect = VMXREFLECTXCPT_XCPT;
5748#ifdef VBOX_STRICT
5749 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5750 && uExitVector == X86_XCPT_PF)
5751 {
5752 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5753 }
5754#endif
5755 if ( uExitVector == X86_XCPT_PF
5756 && uIdtVector == X86_XCPT_PF)
5757 {
5758 pVmxTransient->fVectoringDoublePF = true;
5759 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5760 }
5761 else if ( uExitVector == X86_XCPT_AC
5762 && uIdtVector == X86_XCPT_AC)
5763 {
5764 enmReflect = VMXREFLECTXCPT_HANG;
5765 Log4(("IDT: Nested #AC - Bad guest\n"));
5766 }
5767 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5768 && hmR0VmxIsContributoryXcpt(uExitVector)
5769 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5770 || uIdtVector == X86_XCPT_PF))
5771 {
5772 enmReflect = VMXREFLECTXCPT_DF;
5773 }
5774 else if (uIdtVector == X86_XCPT_DF)
5775 enmReflect = VMXREFLECTXCPT_TF;
5776 }
5777 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5778 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5779 {
5780 /*
5781 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5782 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5783 */
5784 enmReflect = VMXREFLECTXCPT_XCPT;
5785
5786 if (uExitVector == X86_XCPT_PF)
5787 {
5788 pVmxTransient->fVectoringPF = true;
5789 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5790 }
5791 }
5792 }
5793 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5794 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5795 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5796 {
5797 /*
5798 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5799 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
5800 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
5801 */
5802 enmReflect = VMXREFLECTXCPT_XCPT;
5803 }
5804
5805 /*
5806 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
5807 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
5808 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
5809 *
5810 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5811 */
5812 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5813 && enmReflect == VMXREFLECTXCPT_XCPT
5814 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
5815 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5816 {
5817 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5818 }
5819
5820 switch (enmReflect)
5821 {
5822 case VMXREFLECTXCPT_XCPT:
5823 {
5824 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5825 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5826 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5827
5828 uint32_t u32ErrCode = 0;
5829 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5830 {
5831 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5832 AssertRCReturn(rc2, rc2);
5833 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5834 }
5835
5836 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5837 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5838 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5839 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5840 rcStrict = VINF_SUCCESS;
5841 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5842 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5843
5844 break;
5845 }
5846
5847 case VMXREFLECTXCPT_DF:
5848 {
5849 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5850 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5851 rcStrict = VINF_HM_DOUBLE_FAULT;
5852 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5853 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5854
5855 break;
5856 }
5857
5858 case VMXREFLECTXCPT_TF:
5859 {
5860 rcStrict = VINF_EM_RESET;
5861 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5862 uExitVector));
5863 break;
5864 }
5865
5866 case VMXREFLECTXCPT_HANG:
5867 {
5868 rcStrict = VERR_EM_GUEST_CPU_HANG;
5869 break;
5870 }
5871
5872 default:
5873 Assert(rcStrict == VINF_SUCCESS);
5874 break;
5875 }
5876 }
5877 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5878 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5879 && uExitVector != X86_XCPT_DF
5880 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5881 {
5882 /*
5883 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5884 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5885 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5886 */
5887 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5888 {
5889 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
5890 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5891 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5892 }
5893 }
5894
5895 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5896 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5897 return rcStrict;
5898}
5899
5900
5901/**
5902 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5903 *
5904 * @returns VBox status code.
5905 * @param pVCpu The cross context virtual CPU structure.
5906 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5907 * out-of-sync. Make sure to update the required fields
5908 * before using them.
5909 *
5910 * @remarks No-long-jump zone!!!
5911 */
5912static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5913{
5914 NOREF(pMixedCtx);
5915
5916 /*
5917 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5918 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5919 */
5920 VMMRZCallRing3Disable(pVCpu);
5921 HM_DISABLE_PREEMPT();
5922
5923 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5924 {
5925 uint32_t uVal = 0;
5926 uint32_t uShadow = 0;
5927 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5928 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5929 AssertRCReturn(rc, rc);
5930
5931 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5932 CPUMSetGuestCR0(pVCpu, uVal);
5933 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5934 }
5935
5936 HM_RESTORE_PREEMPT();
5937 VMMRZCallRing3Enable(pVCpu);
5938 return VINF_SUCCESS;
5939}
5940
5941
5942/**
5943 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5944 *
5945 * @returns VBox status code.
5946 * @param pVCpu The cross context virtual CPU structure.
5947 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5948 * out-of-sync. Make sure to update the required fields
5949 * before using them.
5950 *
5951 * @remarks No-long-jump zone!!!
5952 */
5953static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5954{
5955 NOREF(pMixedCtx);
5956
5957 int rc = VINF_SUCCESS;
5958 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5959 {
5960 uint32_t uVal = 0;
5961 uint32_t uShadow = 0;
5962 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5963 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5964 AssertRCReturn(rc, rc);
5965
5966 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5967 CPUMSetGuestCR4(pVCpu, uVal);
5968 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5969 }
5970 return rc;
5971}
5972
5973
5974/**
5975 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5976 *
5977 * @returns VBox status code.
5978 * @param pVCpu The cross context virtual CPU structure.
5979 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5980 * out-of-sync. Make sure to update the required fields
5981 * before using them.
5982 *
5983 * @remarks No-long-jump zone!!!
5984 */
5985static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5986{
5987 int rc = VINF_SUCCESS;
5988 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5989 {
5990 uint64_t u64Val = 0;
5991 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5992 AssertRCReturn(rc, rc);
5993
5994 pMixedCtx->rip = u64Val;
5995 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5996 }
5997 return rc;
5998}
5999
6000
6001/**
6002 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6003 *
6004 * @returns VBox status code.
6005 * @param pVCpu The cross context virtual CPU structure.
6006 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6007 * out-of-sync. Make sure to update the required fields
6008 * before using them.
6009 *
6010 * @remarks No-long-jump zone!!!
6011 */
6012static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6013{
6014 int rc = VINF_SUCCESS;
6015 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6016 {
6017 uint64_t u64Val = 0;
6018 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6019 AssertRCReturn(rc, rc);
6020
6021 pMixedCtx->rsp = u64Val;
6022 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6023 }
6024 return rc;
6025}
6026
6027
6028/**
6029 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6030 *
6031 * @returns VBox status code.
6032 * @param pVCpu The cross context virtual CPU structure.
6033 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6034 * out-of-sync. Make sure to update the required fields
6035 * before using them.
6036 *
6037 * @remarks No-long-jump zone!!!
6038 */
6039static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6040{
6041 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6042 {
6043 uint32_t uVal = 0;
6044 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6045 AssertRCReturn(rc, rc);
6046
6047 pMixedCtx->eflags.u32 = uVal;
6048 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6049 {
6050 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6051 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6052
6053 pMixedCtx->eflags.Bits.u1VM = 0;
6054 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6055 }
6056
6057 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6058 }
6059 return VINF_SUCCESS;
6060}
6061
6062
6063/**
6064 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6065 * guest-CPU context.
6066 */
6067DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6068{
6069 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6070 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6071 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6072 return rc;
6073}
6074
6075
6076/**
6077 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6078 * from the guest-state area in the VMCS.
6079 *
6080 * @param pVCpu The cross context virtual CPU structure.
6081 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6082 * out-of-sync. Make sure to update the required fields
6083 * before using them.
6084 *
6085 * @remarks No-long-jump zone!!!
6086 */
6087static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6088{
6089 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6090 {
6091 uint32_t uIntrState = 0;
6092 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6093 AssertRC(rc);
6094
6095 if (!uIntrState)
6096 {
6097 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6098 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6099
6100 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6101 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6102 }
6103 else
6104 {
6105 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6106 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6107 {
6108 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6109 AssertRC(rc);
6110 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6111 AssertRC(rc);
6112
6113 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6114 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6115 }
6116 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6117 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6118
6119 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6120 {
6121 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6122 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6123 }
6124 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6125 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6126 }
6127
6128 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6129 }
6130}
6131
6132
6133/**
6134 * Saves the guest's activity state.
6135 *
6136 * @returns VBox status code.
6137 * @param pVCpu The cross context virtual CPU structure.
6138 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6139 * out-of-sync. Make sure to update the required fields
6140 * before using them.
6141 *
6142 * @remarks No-long-jump zone!!!
6143 */
6144static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6145{
6146 NOREF(pMixedCtx);
6147 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6148 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6149 return VINF_SUCCESS;
6150}
6151
6152
6153/**
6154 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6155 * the current VMCS into the guest-CPU context.
6156 *
6157 * @returns VBox status code.
6158 * @param pVCpu The cross context virtual CPU structure.
6159 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6160 * out-of-sync. Make sure to update the required fields
6161 * before using them.
6162 *
6163 * @remarks No-long-jump zone!!!
6164 */
6165static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6166{
6167 int rc = VINF_SUCCESS;
6168 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6169 {
6170 uint32_t u32Val = 0;
6171 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6172 pMixedCtx->SysEnter.cs = u32Val;
6173 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6174 }
6175
6176 uint64_t u64Val = 0;
6177 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6178 {
6179 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6180 pMixedCtx->SysEnter.eip = u64Val;
6181 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6182 }
6183 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6184 {
6185 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6186 pMixedCtx->SysEnter.esp = u64Val;
6187 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6188 }
6189 return rc;
6190}
6191
6192
6193/**
6194 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6195 * the CPU back into the guest-CPU context.
6196 *
6197 * @returns VBox status code.
6198 * @param pVCpu The cross context virtual CPU structure.
6199 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6200 * out-of-sync. Make sure to update the required fields
6201 * before using them.
6202 *
6203 * @remarks No-long-jump zone!!!
6204 */
6205static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6206{
6207 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6208 VMMRZCallRing3Disable(pVCpu);
6209 HM_DISABLE_PREEMPT();
6210
6211 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6212 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6213 {
6214 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6215 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6216 }
6217
6218 HM_RESTORE_PREEMPT();
6219 VMMRZCallRing3Enable(pVCpu);
6220
6221 return VINF_SUCCESS;
6222}
6223
6224
6225/**
6226 * Saves the auto load/store'd guest MSRs from the current VMCS into
6227 * the guest-CPU context.
6228 *
6229 * @returns VBox status code.
6230 * @param pVCpu The cross context virtual CPU structure.
6231 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6232 * out-of-sync. Make sure to update the required fields
6233 * before using them.
6234 *
6235 * @remarks No-long-jump zone!!!
6236 */
6237static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6238{
6239 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6240 return VINF_SUCCESS;
6241
6242 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6243 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6244 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6245 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6246 {
6247 switch (pMsr->u32Msr)
6248 {
6249 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6250 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6251 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6252 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6253 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6254 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6255 break;
6256
6257 default:
6258 {
6259 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6260 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6261 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6262 }
6263 }
6264 }
6265
6266 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6267 return VINF_SUCCESS;
6268}
6269
6270
6271/**
6272 * Saves the guest control registers from the current VMCS into the guest-CPU
6273 * context.
6274 *
6275 * @returns VBox status code.
6276 * @param pVCpu The cross context virtual CPU structure.
6277 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6278 * out-of-sync. Make sure to update the required fields
6279 * before using them.
6280 *
6281 * @remarks No-long-jump zone!!!
6282 */
6283static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6284{
6285 /* Guest CR0. Guest FPU. */
6286 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6287 AssertRCReturn(rc, rc);
6288
6289 /* Guest CR4. */
6290 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6291 AssertRCReturn(rc, rc);
6292
6293 /* Guest CR2 - updated always during the world-switch or in #PF. */
6294 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6295 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6296 {
6297 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6298 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6299
6300 PVM pVM = pVCpu->CTX_SUFF(pVM);
6301 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6302 || ( pVM->hm.s.fNestedPaging
6303 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6304 {
6305 uint64_t u64Val = 0;
6306 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6307 if (pMixedCtx->cr3 != u64Val)
6308 {
6309 CPUMSetGuestCR3(pVCpu, u64Val);
6310 if (VMMRZCallRing3IsEnabled(pVCpu))
6311 {
6312 PGMUpdateCR3(pVCpu, u64Val);
6313 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6314 }
6315 else
6316 {
6317 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6318 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6319 }
6320 }
6321
6322 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6323 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6324 {
6325 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6326 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6327 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6328 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6329 AssertRCReturn(rc, rc);
6330
6331 if (VMMRZCallRing3IsEnabled(pVCpu))
6332 {
6333 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6334 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6335 }
6336 else
6337 {
6338 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6339 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6340 }
6341 }
6342 }
6343
6344 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6345 }
6346
6347 /*
6348 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6349 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6350 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6351 *
6352 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6353 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6354 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6355 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6356 *
6357 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6358 */
6359 if (VMMRZCallRing3IsEnabled(pVCpu))
6360 {
6361 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6362 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6363
6364 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6365 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6366
6367 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6368 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6369 }
6370
6371 return rc;
6372}
6373
6374
6375/**
6376 * Reads a guest segment register from the current VMCS into the guest-CPU
6377 * context.
6378 *
6379 * @returns VBox status code.
6380 * @param pVCpu The cross context virtual CPU structure.
6381 * @param idxSel Index of the selector in the VMCS.
6382 * @param idxLimit Index of the segment limit in the VMCS.
6383 * @param idxBase Index of the segment base in the VMCS.
6384 * @param idxAccess Index of the access rights of the segment in the VMCS.
6385 * @param pSelReg Pointer to the segment selector.
6386 *
6387 * @remarks No-long-jump zone!!!
6388 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6389 * macro as that takes care of whether to read from the VMCS cache or
6390 * not.
6391 */
6392DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6393 PCPUMSELREG pSelReg)
6394{
6395 NOREF(pVCpu);
6396
6397 uint32_t u32Val = 0;
6398 int rc = VMXReadVmcs32(idxSel, &u32Val);
6399 AssertRCReturn(rc, rc);
6400 pSelReg->Sel = (uint16_t)u32Val;
6401 pSelReg->ValidSel = (uint16_t)u32Val;
6402 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6403
6404 rc = VMXReadVmcs32(idxLimit, &u32Val);
6405 AssertRCReturn(rc, rc);
6406 pSelReg->u32Limit = u32Val;
6407
6408 uint64_t u64Val = 0;
6409 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6410 AssertRCReturn(rc, rc);
6411 pSelReg->u64Base = u64Val;
6412
6413 rc = VMXReadVmcs32(idxAccess, &u32Val);
6414 AssertRCReturn(rc, rc);
6415 pSelReg->Attr.u = u32Val;
6416
6417 /*
6418 * If VT-x marks the segment as unusable, most other bits remain undefined:
6419 * - For CS the L, D and G bits have meaning.
6420 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6421 * - For the remaining data segments no bits are defined.
6422 *
6423 * The present bit and the unusable bit has been observed to be set at the
6424 * same time (the selector was supposed to be invalid as we started executing
6425 * a V8086 interrupt in ring-0).
6426 *
6427 * What should be important for the rest of the VBox code, is that the P bit is
6428 * cleared. Some of the other VBox code recognizes the unusable bit, but
6429 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6430 * safe side here, we'll strip off P and other bits we don't care about. If
6431 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6432 *
6433 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6434 */
6435 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6436 {
6437 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6438
6439 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6440 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6441 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6442
6443 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6444#ifdef DEBUG_bird
6445 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6446 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6447 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6448#endif
6449 }
6450 return VINF_SUCCESS;
6451}
6452
6453
6454#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6455# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6456 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6457 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6458#else
6459# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6460 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6461 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6462#endif
6463
6464
6465/**
6466 * Saves the guest segment registers from the current VMCS into the guest-CPU
6467 * context.
6468 *
6469 * @returns VBox status code.
6470 * @param pVCpu The cross context virtual CPU structure.
6471 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6472 * out-of-sync. Make sure to update the required fields
6473 * before using them.
6474 *
6475 * @remarks No-long-jump zone!!!
6476 */
6477static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6478{
6479 /* Guest segment registers. */
6480 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6481 {
6482 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6483 AssertRCReturn(rc, rc);
6484
6485 rc = VMXLOCAL_READ_SEG(CS, cs);
6486 rc |= VMXLOCAL_READ_SEG(SS, ss);
6487 rc |= VMXLOCAL_READ_SEG(DS, ds);
6488 rc |= VMXLOCAL_READ_SEG(ES, es);
6489 rc |= VMXLOCAL_READ_SEG(FS, fs);
6490 rc |= VMXLOCAL_READ_SEG(GS, gs);
6491 AssertRCReturn(rc, rc);
6492
6493 /* Restore segment attributes for real-on-v86 mode hack. */
6494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6495 {
6496 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6497 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6498 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6499 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6500 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6501 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6502 }
6503 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6504 }
6505
6506 return VINF_SUCCESS;
6507}
6508
6509
6510/**
6511 * Saves the guest descriptor table registers and task register from the current
6512 * VMCS into the guest-CPU context.
6513 *
6514 * @returns VBox status code.
6515 * @param pVCpu The cross context virtual CPU structure.
6516 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6517 * out-of-sync. Make sure to update the required fields
6518 * before using them.
6519 *
6520 * @remarks No-long-jump zone!!!
6521 */
6522static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6523{
6524 int rc = VINF_SUCCESS;
6525
6526 /* Guest LDTR. */
6527 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6528 {
6529 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6530 AssertRCReturn(rc, rc);
6531 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6532 }
6533
6534 /* Guest GDTR. */
6535 uint64_t u64Val = 0;
6536 uint32_t u32Val = 0;
6537 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6538 {
6539 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6540 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6541 pMixedCtx->gdtr.pGdt = u64Val;
6542 pMixedCtx->gdtr.cbGdt = u32Val;
6543 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6544 }
6545
6546 /* Guest IDTR. */
6547 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6548 {
6549 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6550 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6551 pMixedCtx->idtr.pIdt = u64Val;
6552 pMixedCtx->idtr.cbIdt = u32Val;
6553 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6554 }
6555
6556 /* Guest TR. */
6557 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6558 {
6559 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6560 AssertRCReturn(rc, rc);
6561
6562 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6563 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6564 {
6565 rc = VMXLOCAL_READ_SEG(TR, tr);
6566 AssertRCReturn(rc, rc);
6567 }
6568 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6569 }
6570 return rc;
6571}
6572
6573#undef VMXLOCAL_READ_SEG
6574
6575
6576/**
6577 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6578 * context.
6579 *
6580 * @returns VBox status code.
6581 * @param pVCpu The cross context virtual CPU structure.
6582 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6583 * out-of-sync. Make sure to update the required fields
6584 * before using them.
6585 *
6586 * @remarks No-long-jump zone!!!
6587 */
6588static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589{
6590 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6591 {
6592 if (!pVCpu->hm.s.fUsingHyperDR7)
6593 {
6594 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6595 uint32_t u32Val;
6596 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6597 pMixedCtx->dr[7] = u32Val;
6598 }
6599
6600 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6601 }
6602 return VINF_SUCCESS;
6603}
6604
6605
6606/**
6607 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6608 *
6609 * @returns VBox status code.
6610 * @param pVCpu The cross context virtual CPU structure.
6611 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6612 * out-of-sync. Make sure to update the required fields
6613 * before using them.
6614 *
6615 * @remarks No-long-jump zone!!!
6616 */
6617static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6618{
6619 NOREF(pMixedCtx);
6620
6621 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6622 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6623 return VINF_SUCCESS;
6624}
6625
6626
6627/**
6628 * Saves the entire guest state from the currently active VMCS into the
6629 * guest-CPU context.
6630 *
6631 * This essentially VMREADs all guest-data.
6632 *
6633 * @returns VBox status code.
6634 * @param pVCpu The cross context virtual CPU structure.
6635 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6636 * out-of-sync. Make sure to update the required fields
6637 * before using them.
6638 */
6639static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6640{
6641 Assert(pVCpu);
6642 Assert(pMixedCtx);
6643
6644 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6645 return VINF_SUCCESS;
6646
6647 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6648 again on the ring-3 callback path, there is no real need to. */
6649 if (VMMRZCallRing3IsEnabled(pVCpu))
6650 VMMR0LogFlushDisable(pVCpu);
6651 else
6652 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6653 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6654
6655 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6656 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6657
6658 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6659 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6660
6661 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6662 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6663
6664 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6665 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6666
6667 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6668 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6669
6670 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6671 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6672
6673 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6674 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6675
6676 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6677 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6678
6679 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6680 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6681
6682 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6683 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6684
6685 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6686 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
6687 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
6688
6689 if (VMMRZCallRing3IsEnabled(pVCpu))
6690 VMMR0LogFlushEnable(pVCpu);
6691
6692 return VINF_SUCCESS;
6693}
6694
6695
6696/**
6697 * Saves basic guest registers needed for IEM instruction execution.
6698 *
6699 * @returns VBox status code (OR-able).
6700 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6701 * @param pMixedCtx Pointer to the CPU context of the guest.
6702 * @param fMemory Whether the instruction being executed operates on
6703 * memory or not. Only CR0 is synced up if clear.
6704 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
6705 */
6706static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
6707{
6708 /*
6709 * We assume all general purpose registers other than RSP are available.
6710 *
6711 * RIP is a must, as it will be incremented or otherwise changed.
6712 *
6713 * RFLAGS are always required to figure the CPL.
6714 *
6715 * RSP isn't always required, however it's a GPR, so frequently required.
6716 *
6717 * SS and CS are the only segment register needed if IEM doesn't do memory
6718 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
6719 *
6720 * CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
6721 * be required for memory accesses.
6722 *
6723 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
6724 */
6725 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6726 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6727 if (fNeedRsp)
6728 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6729 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6730 if (!fMemory)
6731 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6732 else
6733 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6734 AssertRCReturn(rc, rc);
6735 return rc;
6736}
6737
6738
6739/**
6740 * Ensures that we've got a complete basic guest-context.
6741 *
6742 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
6743 * is for the interpreter.
6744 *
6745 * @returns VBox status code.
6746 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
6747 * @param pMixedCtx Pointer to the guest-CPU context which may have data
6748 * needing to be synced in.
6749 * @thread EMT(pVCpu)
6750 */
6751VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6752{
6753 /* Note! Since this is only applicable to VT-x, the implementation is placed
6754 in the VT-x part of the sources instead of the generic stuff. */
6755 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported)
6756 {
6757 /* For now, imply that the caller might change everything too. */
6758 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
6759 return hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6760 }
6761 return VINF_SUCCESS;
6762}
6763
6764
6765/**
6766 * Check per-VM and per-VCPU force flag actions that require us to go back to
6767 * ring-3 for one reason or another.
6768 *
6769 * @returns Strict VBox status code (i.e. informational status codes too)
6770 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6771 * ring-3.
6772 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6773 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6774 * interrupts)
6775 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6776 * all EMTs to be in ring-3.
6777 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6778 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6779 * to the EM loop.
6780 *
6781 * @param pVM The cross context VM structure.
6782 * @param pVCpu The cross context virtual CPU structure.
6783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6784 * out-of-sync. Make sure to update the required fields
6785 * before using them.
6786 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6787 */
6788static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6789{
6790 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6791
6792 /*
6793 * Anything pending? Should be more likely than not if we're doing a good job.
6794 */
6795 if ( !fStepping
6796 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6797 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6798 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6799 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6800 return VINF_SUCCESS;
6801
6802 /* We need the control registers now, make sure the guest-CPU context is updated. */
6803 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6804 AssertRCReturn(rc3, rc3);
6805
6806 /* Pending HM CR3 sync. */
6807 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6808 {
6809 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6810 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6811 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6812 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6813 }
6814
6815 /* Pending HM PAE PDPEs. */
6816 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6817 {
6818 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6819 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6820 }
6821
6822 /* Pending PGM C3 sync. */
6823 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6824 {
6825 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6826 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6827 if (rcStrict2 != VINF_SUCCESS)
6828 {
6829 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6830 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6831 return rcStrict2;
6832 }
6833 }
6834
6835 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6836 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6837 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6838 {
6839 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6840 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6841 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6842 return rc2;
6843 }
6844
6845 /* Pending VM request packets, such as hardware interrupts. */
6846 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6847 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6848 {
6849 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6850 return VINF_EM_PENDING_REQUEST;
6851 }
6852
6853 /* Pending PGM pool flushes. */
6854 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6855 {
6856 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6857 return VINF_PGM_POOL_FLUSH_PENDING;
6858 }
6859
6860 /* Pending DMA requests. */
6861 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6862 {
6863 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6864 return VINF_EM_RAW_TO_R3;
6865 }
6866
6867 return VINF_SUCCESS;
6868}
6869
6870
6871/**
6872 * Converts any TRPM trap into a pending HM event. This is typically used when
6873 * entering from ring-3 (not longjmp returns).
6874 *
6875 * @param pVCpu The cross context virtual CPU structure.
6876 */
6877static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6878{
6879 Assert(TRPMHasTrap(pVCpu));
6880 Assert(!pVCpu->hm.s.Event.fPending);
6881
6882 uint8_t uVector;
6883 TRPMEVENT enmTrpmEvent;
6884 RTGCUINT uErrCode;
6885 RTGCUINTPTR GCPtrFaultAddress;
6886 uint8_t cbInstr;
6887
6888 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6889 AssertRC(rc);
6890
6891 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6892 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6893 if (enmTrpmEvent == TRPM_TRAP)
6894 {
6895 switch (uVector)
6896 {
6897 case X86_XCPT_NMI:
6898 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6899 break;
6900
6901 case X86_XCPT_BP:
6902 case X86_XCPT_OF:
6903 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6904 break;
6905
6906 case X86_XCPT_PF:
6907 case X86_XCPT_DF:
6908 case X86_XCPT_TS:
6909 case X86_XCPT_NP:
6910 case X86_XCPT_SS:
6911 case X86_XCPT_GP:
6912 case X86_XCPT_AC:
6913 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6914 /* no break! */
6915 default:
6916 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6917 break;
6918 }
6919 }
6920 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6921 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6922 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6923 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6924 else
6925 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6926
6927 rc = TRPMResetTrap(pVCpu);
6928 AssertRC(rc);
6929 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6930 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6931
6932 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6933}
6934
6935
6936/**
6937 * Converts the pending HM event into a TRPM trap.
6938 *
6939 * @param pVCpu The cross context virtual CPU structure.
6940 */
6941static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6942{
6943 Assert(pVCpu->hm.s.Event.fPending);
6944
6945 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6946 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6947 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6948 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6949
6950 /* If a trap was already pending, we did something wrong! */
6951 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6952
6953 TRPMEVENT enmTrapType;
6954 switch (uVectorType)
6955 {
6956 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6957 enmTrapType = TRPM_HARDWARE_INT;
6958 break;
6959
6960 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6961 enmTrapType = TRPM_SOFTWARE_INT;
6962 break;
6963
6964 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6965 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6966 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6967 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6968 enmTrapType = TRPM_TRAP;
6969 break;
6970
6971 default:
6972 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6973 enmTrapType = TRPM_32BIT_HACK;
6974 break;
6975 }
6976
6977 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6978
6979 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6980 AssertRC(rc);
6981
6982 if (fErrorCodeValid)
6983 TRPMSetErrorCode(pVCpu, uErrorCode);
6984
6985 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6986 && uVector == X86_XCPT_PF)
6987 {
6988 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6989 }
6990 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6991 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6992 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6993 {
6994 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6995 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6996 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6997 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6998 }
6999
7000 /* Clear any pending events from the VMCS. */
7001 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7002 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7003
7004 /* We're now done converting the pending event. */
7005 pVCpu->hm.s.Event.fPending = false;
7006}
7007
7008
7009/**
7010 * Does the necessary state syncing before returning to ring-3 for any reason
7011 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7012 *
7013 * @returns VBox status code.
7014 * @param pVM The cross context VM structure.
7015 * @param pVCpu The cross context virtual CPU structure.
7016 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7017 * be out-of-sync. Make sure to update the required
7018 * fields before using them.
7019 * @param fSaveGuestState Whether to save the guest state or not.
7020 *
7021 * @remarks No-long-jmp zone!!!
7022 */
7023static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7024{
7025 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7026 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7027
7028 RTCPUID idCpu = RTMpCpuId();
7029 Log4Func(("HostCpuId=%u\n", idCpu));
7030
7031 /*
7032 * !!! IMPORTANT !!!
7033 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7034 */
7035
7036 /* Save the guest state if necessary. */
7037 if ( fSaveGuestState
7038 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7039 {
7040 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7041 AssertRCReturn(rc, rc);
7042 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7043 }
7044
7045 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7046 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7047 {
7048 if (fSaveGuestState)
7049 {
7050 /* We shouldn't reload CR0 without saving it first. */
7051 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7052 AssertRCReturn(rc, rc);
7053 }
7054 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7055 }
7056
7057 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7058#ifdef VBOX_STRICT
7059 if (CPUMIsHyperDebugStateActive(pVCpu))
7060 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7061#endif
7062 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7063 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7064 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7065 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7066
7067#if HC_ARCH_BITS == 64
7068 /* Restore host-state bits that VT-x only restores partially. */
7069 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7070 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7071 {
7072 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7073 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7074 }
7075 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7076#endif
7077
7078 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7079 if (pVCpu->hm.s.vmx.fLazyMsrs)
7080 {
7081 /* We shouldn't reload the guest MSRs without saving it first. */
7082 if (!fSaveGuestState)
7083 {
7084 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7085 AssertRCReturn(rc, rc);
7086 }
7087 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7088 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7089 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7090 }
7091
7092 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7093 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7094
7095 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7096 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7097 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7098 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7099 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7100 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7101 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7102 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7103
7104 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7105
7106 /** @todo This partially defeats the purpose of having preemption hooks.
7107 * The problem is, deregistering the hooks should be moved to a place that
7108 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7109 * context.
7110 */
7111 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7112 {
7113 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7114 AssertRCReturn(rc, rc);
7115
7116 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7117 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7118 }
7119 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7120 NOREF(idCpu);
7121
7122 return VINF_SUCCESS;
7123}
7124
7125
7126/**
7127 * Leaves the VT-x session.
7128 *
7129 * @returns VBox status code.
7130 * @param pVM The cross context VM structure.
7131 * @param pVCpu The cross context virtual CPU structure.
7132 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7133 * out-of-sync. Make sure to update the required fields
7134 * before using them.
7135 *
7136 * @remarks No-long-jmp zone!!!
7137 */
7138DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7139{
7140 HM_DISABLE_PREEMPT();
7141 HMVMX_ASSERT_CPU_SAFE();
7142 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7143 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7144
7145 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7146 and done this from the VMXR0ThreadCtxCallback(). */
7147 if (!pVCpu->hm.s.fLeaveDone)
7148 {
7149 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7150 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7151 pVCpu->hm.s.fLeaveDone = true;
7152 }
7153 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7154
7155 /*
7156 * !!! IMPORTANT !!!
7157 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7158 */
7159
7160 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7161 /** @todo Deregistering here means we need to VMCLEAR always
7162 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7163 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7164 VMMR0ThreadCtxHookDisable(pVCpu);
7165
7166 /* Leave HM context. This takes care of local init (term). */
7167 int rc = HMR0LeaveCpu(pVCpu);
7168
7169 HM_RESTORE_PREEMPT();
7170 return rc;
7171}
7172
7173
7174/**
7175 * Does the necessary state syncing before doing a longjmp to ring-3.
7176 *
7177 * @returns VBox status code.
7178 * @param pVM The cross context VM structure.
7179 * @param pVCpu The cross context virtual CPU structure.
7180 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7181 * out-of-sync. Make sure to update the required fields
7182 * before using them.
7183 *
7184 * @remarks No-long-jmp zone!!!
7185 */
7186DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7187{
7188 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7189}
7190
7191
7192/**
7193 * Take necessary actions before going back to ring-3.
7194 *
7195 * An action requires us to go back to ring-3. This function does the necessary
7196 * steps before we can safely return to ring-3. This is not the same as longjmps
7197 * to ring-3, this is voluntary and prepares the guest so it may continue
7198 * executing outside HM (recompiler/IEM).
7199 *
7200 * @returns VBox status code.
7201 * @param pVM The cross context VM structure.
7202 * @param pVCpu The cross context virtual CPU structure.
7203 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7204 * out-of-sync. Make sure to update the required fields
7205 * before using them.
7206 * @param rcExit The reason for exiting to ring-3. Can be
7207 * VINF_VMM_UNKNOWN_RING3_CALL.
7208 */
7209static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7210{
7211 Assert(pVM);
7212 Assert(pVCpu);
7213 Assert(pMixedCtx);
7214 HMVMX_ASSERT_PREEMPT_SAFE();
7215
7216 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7217 {
7218 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7219 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7220 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7221 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7222 }
7223
7224 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7225 VMMRZCallRing3Disable(pVCpu);
7226 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7227
7228 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7229 if (pVCpu->hm.s.Event.fPending)
7230 {
7231 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7232 Assert(!pVCpu->hm.s.Event.fPending);
7233 }
7234
7235 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7236 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7237
7238 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7239 and if we're injecting an event we should have a TRPM trap pending. */
7240 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7241#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7242 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7243#endif
7244
7245 /* Save guest state and restore host state bits. */
7246 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7247 AssertRCReturn(rc, rc);
7248 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7249 /* Thread-context hooks are unregistered at this point!!! */
7250
7251 /* Sync recompiler state. */
7252 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7253 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7254 | CPUM_CHANGED_LDTR
7255 | CPUM_CHANGED_GDTR
7256 | CPUM_CHANGED_IDTR
7257 | CPUM_CHANGED_TR
7258 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7259 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7260 if ( pVM->hm.s.fNestedPaging
7261 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7262 {
7263 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7264 }
7265
7266 Assert(!pVCpu->hm.s.fClearTrapFlag);
7267
7268 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7269 if (rcExit != VINF_EM_RAW_INTERRUPT)
7270 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7271
7272 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7273
7274 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7275 VMMRZCallRing3RemoveNotification(pVCpu);
7276 VMMRZCallRing3Enable(pVCpu);
7277
7278 return rc;
7279}
7280
7281
7282/**
7283 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7284 * longjump to ring-3 and possibly get preempted.
7285 *
7286 * @returns VBox status code.
7287 * @param pVCpu The cross context virtual CPU structure.
7288 * @param enmOperation The operation causing the ring-3 longjump.
7289 * @param pvUser Opaque pointer to the guest-CPU context. The data
7290 * may be out-of-sync. Make sure to update the required
7291 * fields before using them.
7292 */
7293static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7294{
7295 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7296 {
7297 /*
7298 * !!! IMPORTANT !!!
7299 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7300 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7301 */
7302 VMMRZCallRing3RemoveNotification(pVCpu);
7303 VMMRZCallRing3Disable(pVCpu);
7304 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7305 RTThreadPreemptDisable(&PreemptState);
7306
7307 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7308 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7309
7310#if HC_ARCH_BITS == 64
7311 /* Restore host-state bits that VT-x only restores partially. */
7312 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7313 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7314 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7315 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7316#endif
7317 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7318 if (pVCpu->hm.s.vmx.fLazyMsrs)
7319 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7320
7321 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7322 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7323 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7324 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7325 {
7326 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7327 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7328 }
7329
7330 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7331 VMMR0ThreadCtxHookDisable(pVCpu);
7332 HMR0LeaveCpu(pVCpu);
7333 RTThreadPreemptRestore(&PreemptState);
7334 return VINF_SUCCESS;
7335 }
7336
7337 Assert(pVCpu);
7338 Assert(pvUser);
7339 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7340 HMVMX_ASSERT_PREEMPT_SAFE();
7341
7342 VMMRZCallRing3Disable(pVCpu);
7343 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7344
7345 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7346 enmOperation));
7347
7348 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7349 AssertRCReturn(rc, rc);
7350
7351 VMMRZCallRing3Enable(pVCpu);
7352 return VINF_SUCCESS;
7353}
7354
7355
7356/**
7357 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7358 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7359 *
7360 * @param pVCpu The cross context virtual CPU structure.
7361 */
7362DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7363{
7364 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7365 {
7366 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7367 {
7368 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7369 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7370 AssertRC(rc);
7371 Log4(("Setup interrupt-window exiting\n"));
7372 }
7373 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7374}
7375
7376
7377/**
7378 * Clears the interrupt-window exiting control in the VMCS.
7379 *
7380 * @param pVCpu The cross context virtual CPU structure.
7381 */
7382DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7383{
7384 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7385 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7386 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7387 AssertRC(rc);
7388 Log4(("Cleared interrupt-window exiting\n"));
7389}
7390
7391
7392/**
7393 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7394 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7395 *
7396 * @param pVCpu The cross context virtual CPU structure.
7397 */
7398DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7399{
7400 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7401 {
7402 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7403 {
7404 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7405 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7406 AssertRC(rc);
7407 Log4(("Setup NMI-window exiting\n"));
7408 }
7409 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7410}
7411
7412
7413/**
7414 * Clears the NMI-window exiting control in the VMCS.
7415 *
7416 * @param pVCpu The cross context virtual CPU structure.
7417 */
7418DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7419{
7420 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7421 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7422 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7423 AssertRC(rc);
7424 Log4(("Cleared NMI-window exiting\n"));
7425}
7426
7427
7428/**
7429 * Evaluates the event to be delivered to the guest and sets it as the pending
7430 * event.
7431 *
7432 * @returns The VT-x guest-interruptibility state.
7433 * @param pVCpu The cross context virtual CPU structure.
7434 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7435 * out-of-sync. Make sure to update the required fields
7436 * before using them.
7437 */
7438static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7439{
7440 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7441 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7442 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7443 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7444 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7445
7446 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7447 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7448 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7449 Assert(!TRPMHasTrap(pVCpu));
7450
7451#ifdef VBOX_WITH_NEW_APIC
7452 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7453 APICUpdatePendingInterrupts(pVCpu);
7454#endif
7455
7456 /*
7457 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7458 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7459 */
7460 /** @todo SMI. SMIs take priority over NMIs. */
7461 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7462 {
7463 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7464 if ( !pVCpu->hm.s.Event.fPending
7465 && !fBlockNmi
7466 && !fBlockSti
7467 && !fBlockMovSS)
7468 {
7469 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7470 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7471 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7472
7473 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7474 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7475 }
7476 else
7477 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7478 }
7479 /*
7480 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7481 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7482 */
7483 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7484 && !pVCpu->hm.s.fSingleInstruction)
7485 {
7486 Assert(!DBGFIsStepping(pVCpu));
7487 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7488 AssertRC(rc);
7489 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7490 if ( !pVCpu->hm.s.Event.fPending
7491 && !fBlockInt
7492 && !fBlockSti
7493 && !fBlockMovSS)
7494 {
7495 uint8_t u8Interrupt;
7496 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7497 if (RT_SUCCESS(rc))
7498 {
7499 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7500 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7501 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7502
7503 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7504 }
7505 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7506 {
7507 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7508 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7509 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7510 }
7511 else
7512 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7513 }
7514 else
7515 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7516 }
7517
7518 return uIntrState;
7519}
7520
7521
7522/**
7523 * Sets a pending-debug exception to be delivered to the guest if the guest is
7524 * single-stepping in the VMCS.
7525 *
7526 * @param pVCpu The cross context virtual CPU structure.
7527 */
7528DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7529{
7530 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7531 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7532 AssertRC(rc);
7533}
7534
7535
7536/**
7537 * Injects any pending events into the guest if the guest is in a state to
7538 * receive them.
7539 *
7540 * @returns Strict VBox status code (i.e. informational status codes too).
7541 * @param pVCpu The cross context virtual CPU structure.
7542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7543 * out-of-sync. Make sure to update the required fields
7544 * before using them.
7545 * @param uIntrState The VT-x guest-interruptibility state.
7546 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7547 * return VINF_EM_DBG_STEPPED if the event was
7548 * dispatched directly.
7549 */
7550static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7551{
7552 HMVMX_ASSERT_PREEMPT_SAFE();
7553 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7554
7555 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7556 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7557
7558 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7559 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7560 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7561 Assert(!TRPMHasTrap(pVCpu));
7562
7563 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7564 if (pVCpu->hm.s.Event.fPending)
7565 {
7566 /*
7567 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7568 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7569 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7570 *
7571 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7572 */
7573 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7574#ifdef VBOX_STRICT
7575 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7576 {
7577 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7578 Assert(!fBlockInt);
7579 Assert(!fBlockSti);
7580 Assert(!fBlockMovSS);
7581 }
7582 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7583 {
7584 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7585 Assert(!fBlockSti);
7586 Assert(!fBlockMovSS);
7587 Assert(!fBlockNmi);
7588 }
7589#endif
7590 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7591 (uint8_t)uIntType));
7592 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7593 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7594 fStepping, &uIntrState);
7595 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7596
7597 /* Update the interruptibility-state as it could have been changed by
7598 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7599 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7600 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7601
7602 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7603 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7604 else
7605 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7606 }
7607
7608 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7609 if ( fBlockSti
7610 || fBlockMovSS)
7611 {
7612 if (!pVCpu->hm.s.fSingleInstruction)
7613 {
7614 /*
7615 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7616 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7617 * See Intel spec. 27.3.4 "Saving Non-Register State".
7618 */
7619 Assert(!DBGFIsStepping(pVCpu));
7620 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7621 AssertRCReturn(rc2, rc2);
7622 if (pMixedCtx->eflags.Bits.u1TF)
7623 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7624 }
7625 else if (pMixedCtx->eflags.Bits.u1TF)
7626 {
7627 /*
7628 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7629 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7630 */
7631 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7632 uIntrState = 0;
7633 }
7634 }
7635
7636 /*
7637 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7638 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7639 */
7640 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7641 AssertRC(rc2);
7642
7643 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7644 NOREF(fBlockMovSS); NOREF(fBlockSti);
7645 return rcStrict;
7646}
7647
7648
7649/**
7650 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7651 *
7652 * @param pVCpu The cross context virtual CPU structure.
7653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7654 * out-of-sync. Make sure to update the required fields
7655 * before using them.
7656 */
7657DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7658{
7659 NOREF(pMixedCtx);
7660 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7661 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7662}
7663
7664
7665/**
7666 * Injects a double-fault (\#DF) exception into the VM.
7667 *
7668 * @returns Strict VBox status code (i.e. informational status codes too).
7669 * @param pVCpu The cross context virtual CPU structure.
7670 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7671 * out-of-sync. Make sure to update the required fields
7672 * before using them.
7673 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7674 * and should return VINF_EM_DBG_STEPPED if the event
7675 * is injected directly (register modified by us, not
7676 * by hardware on VM-entry).
7677 * @param puIntrState Pointer to the current guest interruptibility-state.
7678 * This interruptibility-state will be updated if
7679 * necessary. This cannot not be NULL.
7680 */
7681DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
7682{
7683 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7684 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7685 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7686 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7687 fStepping, puIntrState);
7688}
7689
7690
7691/**
7692 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7693 *
7694 * @param pVCpu The cross context virtual CPU structure.
7695 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7696 * out-of-sync. Make sure to update the required fields
7697 * before using them.
7698 */
7699DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7700{
7701 NOREF(pMixedCtx);
7702 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7703 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7704 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7705}
7706
7707
7708/**
7709 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7710 *
7711 * @param pVCpu The cross context virtual CPU structure.
7712 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7713 * out-of-sync. Make sure to update the required fields
7714 * before using them.
7715 * @param cbInstr The value of RIP that is to be pushed on the guest
7716 * stack.
7717 */
7718DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7719{
7720 NOREF(pMixedCtx);
7721 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7722 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7723 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7724}
7725
7726
7727/**
7728 * Injects a general-protection (\#GP) fault into the VM.
7729 *
7730 * @returns Strict VBox status code (i.e. informational status codes too).
7731 * @param pVCpu The cross context virtual CPU structure.
7732 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7733 * out-of-sync. Make sure to update the required fields
7734 * before using them.
7735 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7736 * mode, i.e. in real-mode it's not valid).
7737 * @param u32ErrorCode The error code associated with the \#GP.
7738 * @param fStepping Whether we're running in
7739 * hmR0VmxRunGuestCodeStep() and should return
7740 * VINF_EM_DBG_STEPPED if the event is injected
7741 * directly (register modified by us, not by
7742 * hardware on VM-entry).
7743 * @param puIntrState Pointer to the current guest interruptibility-state.
7744 * This interruptibility-state will be updated if
7745 * necessary. This cannot not be NULL.
7746 */
7747DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7748 bool fStepping, uint32_t *puIntrState)
7749{
7750 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7751 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7752 if (fErrorCodeValid)
7753 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7754 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7755 fStepping, puIntrState);
7756}
7757
7758
7759/**
7760 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7761 * VM.
7762 *
7763 * @param pVCpu The cross context virtual CPU structure.
7764 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7765 * out-of-sync. Make sure to update the required fields
7766 * before using them.
7767 * @param u32ErrorCode The error code associated with the \#GP.
7768 */
7769DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7770{
7771 NOREF(pMixedCtx);
7772 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7773 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7774 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7775 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7776}
7777
7778
7779/**
7780 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7781 *
7782 * @param pVCpu The cross context virtual CPU structure.
7783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7784 * out-of-sync. Make sure to update the required fields
7785 * before using them.
7786 * @param uVector The software interrupt vector number.
7787 * @param cbInstr The value of RIP that is to be pushed on the guest
7788 * stack.
7789 */
7790DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7791{
7792 NOREF(pMixedCtx);
7793 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7794 if ( uVector == X86_XCPT_BP
7795 || uVector == X86_XCPT_OF)
7796 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7797 else
7798 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7799 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7800}
7801
7802
7803/**
7804 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7805 * stack.
7806 *
7807 * @returns Strict VBox status code (i.e. informational status codes too).
7808 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7809 * @param pVM The cross context VM structure.
7810 * @param pMixedCtx Pointer to the guest-CPU context.
7811 * @param uValue The value to push to the guest stack.
7812 */
7813DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7814{
7815 /*
7816 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7817 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7818 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7819 */
7820 if (pMixedCtx->sp == 1)
7821 return VINF_EM_RESET;
7822 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7823 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7824 AssertRC(rc);
7825 return rc;
7826}
7827
7828
7829/**
7830 * Injects an event into the guest upon VM-entry by updating the relevant fields
7831 * in the VM-entry area in the VMCS.
7832 *
7833 * @returns Strict VBox status code (i.e. informational status codes too).
7834 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7835 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7836 *
7837 * @param pVCpu The cross context virtual CPU structure.
7838 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7839 * be out-of-sync. Make sure to update the required
7840 * fields before using them.
7841 * @param u64IntInfo The VM-entry interruption-information field.
7842 * @param cbInstr The VM-entry instruction length in bytes (for
7843 * software interrupts, exceptions and privileged
7844 * software exceptions).
7845 * @param u32ErrCode The VM-entry exception error code.
7846 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7847 * @param puIntrState Pointer to the current guest interruptibility-state.
7848 * This interruptibility-state will be updated if
7849 * necessary. This cannot not be NULL.
7850 * @param fStepping Whether we're running in
7851 * hmR0VmxRunGuestCodeStep() and should return
7852 * VINF_EM_DBG_STEPPED if the event is injected
7853 * directly (register modified by us, not by
7854 * hardware on VM-entry).
7855 *
7856 * @remarks Requires CR0!
7857 * @remarks No-long-jump zone!!!
7858 */
7859static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7860 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
7861 uint32_t *puIntrState)
7862{
7863 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7864 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7865 Assert(puIntrState);
7866 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7867
7868 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7869 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7870
7871#ifdef VBOX_STRICT
7872 /* Validate the error-code-valid bit for hardware exceptions. */
7873 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7874 {
7875 switch (uVector)
7876 {
7877 case X86_XCPT_PF:
7878 case X86_XCPT_DF:
7879 case X86_XCPT_TS:
7880 case X86_XCPT_NP:
7881 case X86_XCPT_SS:
7882 case X86_XCPT_GP:
7883 case X86_XCPT_AC:
7884 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7885 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7886 /* fallthru */
7887 default:
7888 break;
7889 }
7890 }
7891#endif
7892
7893 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7894 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7895 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7896
7897 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7898
7899 /* We require CR0 to check if the guest is in real-mode. */
7900 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7901 AssertRCReturn(rc, rc);
7902
7903 /*
7904 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7905 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7906 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7907 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7908 */
7909 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7910 {
7911 PVM pVM = pVCpu->CTX_SUFF(pVM);
7912 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7913 {
7914 Assert(PDMVmmDevHeapIsEnabled(pVM));
7915 Assert(pVM->hm.s.vmx.pRealModeTSS);
7916
7917 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7918 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7919 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7920 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7921 AssertRCReturn(rc, rc);
7922 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7923
7924 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7925 size_t const cbIdtEntry = sizeof(X86IDTR16);
7926 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7927 {
7928 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7929 if (uVector == X86_XCPT_DF)
7930 return VINF_EM_RESET;
7931
7932 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7933 if (uVector == X86_XCPT_GP)
7934 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
7935
7936 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7937 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7938 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
7939 fStepping, puIntrState);
7940 }
7941
7942 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7943 uint16_t uGuestIp = pMixedCtx->ip;
7944 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7945 {
7946 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7947 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7948 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7949 }
7950 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7951 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7952
7953 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7954 X86IDTR16 IdtEntry;
7955 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7956 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7957 AssertRCReturn(rc, rc);
7958
7959 /* Construct the stack frame for the interrupt/exception handler. */
7960 VBOXSTRICTRC rcStrict;
7961 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7962 if (rcStrict == VINF_SUCCESS)
7963 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7964 if (rcStrict == VINF_SUCCESS)
7965 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7966
7967 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7968 if (rcStrict == VINF_SUCCESS)
7969 {
7970 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7971 pMixedCtx->rip = IdtEntry.offSel;
7972 pMixedCtx->cs.Sel = IdtEntry.uSel;
7973 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7974 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7975 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7976 && uVector == X86_XCPT_PF)
7977 pMixedCtx->cr2 = GCPtrFaultAddress;
7978
7979 /* If any other guest-state bits are changed here, make sure to update
7980 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7981 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7982 | HM_CHANGED_GUEST_RIP
7983 | HM_CHANGED_GUEST_RFLAGS
7984 | HM_CHANGED_GUEST_RSP);
7985
7986 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7987 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7988 {
7989 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7990 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7991 Log4(("Clearing inhibition due to STI.\n"));
7992 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7993 }
7994 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7995 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7996
7997 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7998 it, if we are returning to ring-3 before executing guest code. */
7999 pVCpu->hm.s.Event.fPending = false;
8000
8001 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8002 if (fStepping)
8003 rcStrict = VINF_EM_DBG_STEPPED;
8004 }
8005 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8006 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8007 return rcStrict;
8008 }
8009
8010 /*
8011 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8012 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8013 */
8014 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8015 }
8016
8017 /* Validate. */
8018 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8019 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8020 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8021
8022 /* Inject. */
8023 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8024 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8025 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8026 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8027
8028 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8029 && uVector == X86_XCPT_PF)
8030 pMixedCtx->cr2 = GCPtrFaultAddress;
8031
8032 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8033 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8034
8035 AssertRCReturn(rc, rc);
8036 return VINF_SUCCESS;
8037}
8038
8039
8040/**
8041 * Clears the interrupt-window exiting control in the VMCS and if necessary
8042 * clears the current event in the VMCS as well.
8043 *
8044 * @returns VBox status code.
8045 * @param pVCpu The cross context virtual CPU structure.
8046 *
8047 * @remarks Use this function only to clear events that have not yet been
8048 * delivered to the guest but are injected in the VMCS!
8049 * @remarks No-long-jump zone!!!
8050 */
8051static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8052{
8053 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8054
8055 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8056 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8057
8058 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8059 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8060}
8061
8062
8063/**
8064 * Enters the VT-x session.
8065 *
8066 * @returns VBox status code.
8067 * @param pVM The cross context VM structure.
8068 * @param pVCpu The cross context virtual CPU structure.
8069 * @param pCpu Pointer to the CPU info struct.
8070 */
8071VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8072{
8073 AssertPtr(pVM);
8074 AssertPtr(pVCpu);
8075 Assert(pVM->hm.s.vmx.fSupported);
8076 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8077 NOREF(pCpu); NOREF(pVM);
8078
8079 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8080 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8081
8082#ifdef VBOX_STRICT
8083 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8084 RTCCUINTREG uHostCR4 = ASMGetCR4();
8085 if (!(uHostCR4 & X86_CR4_VMXE))
8086 {
8087 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8088 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8089 }
8090#endif
8091
8092 /*
8093 * Load the VCPU's VMCS as the current (and active) one.
8094 */
8095 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8096 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8097 if (RT_FAILURE(rc))
8098 return rc;
8099
8100 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8101 pVCpu->hm.s.fLeaveDone = false;
8102 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8103
8104 return VINF_SUCCESS;
8105}
8106
8107
8108/**
8109 * The thread-context callback (only on platforms which support it).
8110 *
8111 * @param enmEvent The thread-context event.
8112 * @param pVCpu The cross context virtual CPU structure.
8113 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8114 * @thread EMT(pVCpu)
8115 */
8116VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8117{
8118 NOREF(fGlobalInit);
8119
8120 switch (enmEvent)
8121 {
8122 case RTTHREADCTXEVENT_OUT:
8123 {
8124 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8125 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8126 VMCPU_ASSERT_EMT(pVCpu);
8127
8128 PVM pVM = pVCpu->CTX_SUFF(pVM);
8129 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8130
8131 /* No longjmps (logger flushes, locks) in this fragile context. */
8132 VMMRZCallRing3Disable(pVCpu);
8133 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8134
8135 /*
8136 * Restore host-state (FPU, debug etc.)
8137 */
8138 if (!pVCpu->hm.s.fLeaveDone)
8139 {
8140 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8141 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8142 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8143 pVCpu->hm.s.fLeaveDone = true;
8144 }
8145
8146 /* Leave HM context, takes care of local init (term). */
8147 int rc = HMR0LeaveCpu(pVCpu);
8148 AssertRC(rc); NOREF(rc);
8149
8150 /* Restore longjmp state. */
8151 VMMRZCallRing3Enable(pVCpu);
8152 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8153 break;
8154 }
8155
8156 case RTTHREADCTXEVENT_IN:
8157 {
8158 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8159 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8160 VMCPU_ASSERT_EMT(pVCpu);
8161
8162 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8163 VMMRZCallRing3Disable(pVCpu);
8164 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8165
8166 /* Initialize the bare minimum state required for HM. This takes care of
8167 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8168 int rc = HMR0EnterCpu(pVCpu);
8169 AssertRC(rc);
8170 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8171
8172 /* Load the active VMCS as the current one. */
8173 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8174 {
8175 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8176 AssertRC(rc); NOREF(rc);
8177 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8178 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8179 }
8180 pVCpu->hm.s.fLeaveDone = false;
8181
8182 /* Restore longjmp state. */
8183 VMMRZCallRing3Enable(pVCpu);
8184 break;
8185 }
8186
8187 default:
8188 break;
8189 }
8190}
8191
8192
8193/**
8194 * Saves the host state in the VMCS host-state.
8195 * Sets up the VM-exit MSR-load area.
8196 *
8197 * The CPU state will be loaded from these fields on every successful VM-exit.
8198 *
8199 * @returns VBox status code.
8200 * @param pVM The cross context VM structure.
8201 * @param pVCpu The cross context virtual CPU structure.
8202 *
8203 * @remarks No-long-jump zone!!!
8204 */
8205static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8206{
8207 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8208
8209 int rc = VINF_SUCCESS;
8210 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8211 {
8212 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8213 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8214
8215 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8216 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8217
8218 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8219 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8220
8221 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8222 }
8223 return rc;
8224}
8225
8226
8227/**
8228 * Saves the host state in the VMCS host-state.
8229 *
8230 * @returns VBox status code.
8231 * @param pVM The cross context VM structure.
8232 * @param pVCpu The cross context virtual CPU structure.
8233 *
8234 * @remarks No-long-jump zone!!!
8235 */
8236VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8237{
8238 AssertPtr(pVM);
8239 AssertPtr(pVCpu);
8240
8241 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8242
8243 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8244 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8245 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8246 return hmR0VmxSaveHostState(pVM, pVCpu);
8247}
8248
8249
8250/**
8251 * Loads the guest state into the VMCS guest-state area.
8252 *
8253 * The will typically be done before VM-entry when the guest-CPU state and the
8254 * VMCS state may potentially be out of sync.
8255 *
8256 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8257 * VM-entry controls.
8258 * Sets up the appropriate VMX non-root function to execute guest code based on
8259 * the guest CPU mode.
8260 *
8261 * @returns VBox strict status code.
8262 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8263 * without unrestricted guest access and the VMMDev is not presently
8264 * mapped (e.g. EFI32).
8265 *
8266 * @param pVM The cross context VM structure.
8267 * @param pVCpu The cross context virtual CPU structure.
8268 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8269 * out-of-sync. Make sure to update the required fields
8270 * before using them.
8271 *
8272 * @remarks No-long-jump zone!!! (Disables and enables long jmps for itself,
8273 * caller disables then again on successfull return. Confusing.)
8274 */
8275static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8276{
8277 AssertPtr(pVM);
8278 AssertPtr(pVCpu);
8279 AssertPtr(pMixedCtx);
8280 HMVMX_ASSERT_PREEMPT_SAFE();
8281
8282 VMMRZCallRing3Disable(pVCpu);
8283 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8284
8285 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8286
8287 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8288
8289 /* Determine real-on-v86 mode. */
8290 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8291 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8292 && CPUMIsGuestInRealModeEx(pMixedCtx))
8293 {
8294 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8295 }
8296
8297 /*
8298 * Load the guest-state into the VMCS.
8299 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8300 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8301 */
8302 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8303 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8304
8305 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8306 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8307 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8308
8309 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8310 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8311 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8312
8313 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8314 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8315
8316 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8317 if (rcStrict == VINF_SUCCESS)
8318 { /* likely */ }
8319 else
8320 {
8321 VMMRZCallRing3Enable(pVCpu);
8322 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8323 return rcStrict;
8324 }
8325
8326 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8327 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8328 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8329
8330 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8331 determine we don't have to swap EFER after all. */
8332 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8333 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8334
8335 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8336 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8337
8338 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8339 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8340
8341 /*
8342 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8343 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8344 */
8345 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8346 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8347
8348 /* Clear any unused and reserved bits. */
8349 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8350
8351 VMMRZCallRing3Enable(pVCpu);
8352
8353 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8354 return rc;
8355}
8356
8357
8358/**
8359 * Loads the state shared between the host and guest into the VMCS.
8360 *
8361 * @param pVM The cross context VM structure.
8362 * @param pVCpu The cross context virtual CPU structure.
8363 * @param pCtx Pointer to the guest-CPU context.
8364 *
8365 * @remarks No-long-jump zone!!!
8366 */
8367static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8368{
8369 NOREF(pVM);
8370
8371 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8372 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8373
8374 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8375 {
8376 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8377 AssertRC(rc);
8378 }
8379
8380 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8381 {
8382 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8383 AssertRC(rc);
8384
8385 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8386 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8387 {
8388 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8389 AssertRC(rc);
8390 }
8391 }
8392
8393 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8394 {
8395 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8396 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8397 }
8398
8399 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8400 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS))
8401 {
8402 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8403 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8404 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8405 AssertRC(rc);
8406 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS);
8407 }
8408
8409 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8410 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8411}
8412
8413
8414/**
8415 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8416 *
8417 * @returns Strict VBox status code (i.e. informational status codes too).
8418 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8419 * without unrestricted guest access and the VMMDev is not presently
8420 * mapped (e.g. EFI32).
8421 *
8422 * @param pVM The cross context VM structure.
8423 * @param pVCpu The cross context virtual CPU structure.
8424 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8425 * out-of-sync. Make sure to update the required fields
8426 * before using them.
8427 */
8428static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8429{
8430 HMVMX_ASSERT_PREEMPT_SAFE();
8431
8432 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8433#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8434 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8435#endif
8436
8437 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8438 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8439 {
8440 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8441 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8442 { /* likely */}
8443 else
8444 {
8445 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8446 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8447 }
8448 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8449 }
8450 else if (HMCPU_CF_VALUE(pVCpu))
8451 {
8452 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8453 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8454 { /* likely */}
8455 else
8456 {
8457 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8458 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8459 return rcStrict;
8460 }
8461 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8462 }
8463
8464 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8465 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8466 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8467 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8468 return rcStrict;
8469}
8470
8471
8472/**
8473 * Does the preparations before executing guest code in VT-x.
8474 *
8475 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8476 * recompiler/IEM. We must be cautious what we do here regarding committing
8477 * guest-state information into the VMCS assuming we assuredly execute the
8478 * guest in VT-x mode.
8479 *
8480 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8481 * the common-state (TRPM/forceflags), we must undo those changes so that the
8482 * recompiler/IEM can (and should) use them when it resumes guest execution.
8483 * Otherwise such operations must be done when we can no longer exit to ring-3.
8484 *
8485 * @returns Strict VBox status code (i.e. informational status codes too).
8486 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8487 * have been disabled.
8488 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8489 * double-fault into the guest.
8490 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8491 * dispatched directly.
8492 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8493 *
8494 * @param pVM The cross context VM structure.
8495 * @param pVCpu The cross context virtual CPU structure.
8496 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8497 * out-of-sync. Make sure to update the required fields
8498 * before using them.
8499 * @param pVmxTransient Pointer to the VMX transient structure.
8500 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8501 * us ignore some of the reasons for returning to
8502 * ring-3, and return VINF_EM_DBG_STEPPED if event
8503 * dispatching took place.
8504 */
8505static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8506{
8507 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8508
8509#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8510 PGMRZDynMapFlushAutoSet(pVCpu);
8511#endif
8512
8513 /* Check force flag actions that might require us to go back to ring-3. */
8514 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8515 if (rcStrict == VINF_SUCCESS)
8516 { /* FFs doesn't get set all the time. */ }
8517 else
8518 return rcStrict;
8519
8520#ifndef IEM_VERIFICATION_MODE_FULL
8521 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8522 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8523 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8524 {
8525 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8526 RTGCPHYS GCPhysApicBase;
8527 GCPhysApicBase = pMixedCtx->msrApicBase;
8528 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8529
8530 /* Unalias any existing mapping. */
8531 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8532 AssertRCReturn(rc, rc);
8533
8534 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8535 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGp\n", GCPhysApicBase));
8536 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8537 AssertRCReturn(rc, rc);
8538
8539 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8540 }
8541#endif /* !IEM_VERIFICATION_MODE_FULL */
8542
8543 if (TRPMHasTrap(pVCpu))
8544 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8545 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8546
8547 /*
8548 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8549 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8550 */
8551 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8552 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8553 { /* likely */ }
8554 else
8555 {
8556 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8557 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8558 return rcStrict;
8559 }
8560
8561 /*
8562 * Load the guest state bits, we can handle longjmps/getting preempted here.
8563 *
8564 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8565 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8566 * Hence, this needs to be done -after- injection of events.
8567 */
8568 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8569 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8570 { /* likely */ }
8571 else
8572 return rcStrict;
8573
8574 /*
8575 * No longjmps to ring-3 from this point on!!!
8576 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8577 * This also disables flushing of the R0-logger instance (if any).
8578 */
8579 VMMRZCallRing3Disable(pVCpu);
8580
8581 /*
8582 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8583 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8584 *
8585 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8586 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8587 *
8588 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8589 * executing guest code.
8590 */
8591 pVmxTransient->fEFlags = ASMIntDisableFlags();
8592
8593 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8594 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8595 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8596 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8597 {
8598 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8599 {
8600 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8601 pVCpu->hm.s.Event.fPending = false;
8602
8603 return VINF_SUCCESS;
8604 }
8605
8606 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8607 rcStrict = VINF_EM_RAW_INTERRUPT;
8608 }
8609 else
8610 {
8611 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8612 rcStrict = VINF_EM_RAW_TO_R3;
8613 }
8614
8615 ASMSetFlags(pVmxTransient->fEFlags);
8616 VMMRZCallRing3Enable(pVCpu);
8617
8618 return rcStrict;
8619}
8620
8621
8622/**
8623 * Prepares to run guest code in VT-x and we've committed to doing so. This
8624 * means there is no backing out to ring-3 or anywhere else at this
8625 * point.
8626 *
8627 * @param pVM The cross context VM structure.
8628 * @param pVCpu The cross context virtual CPU structure.
8629 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8630 * out-of-sync. Make sure to update the required fields
8631 * before using them.
8632 * @param pVmxTransient Pointer to the VMX transient structure.
8633 *
8634 * @remarks Called with preemption disabled.
8635 * @remarks No-long-jump zone!!!
8636 */
8637static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8638{
8639 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8640 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8641 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8642
8643 /*
8644 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8645 */
8646 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8647 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8648
8649#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8650 if (!CPUMIsGuestFPUStateActive(pVCpu))
8651 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8652 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8653 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8654#endif
8655
8656 if ( pVCpu->hm.s.fPreloadGuestFpu
8657 && !CPUMIsGuestFPUStateActive(pVCpu))
8658 {
8659 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8660 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
8661 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8662 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8663 }
8664
8665 /*
8666 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8667 */
8668 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8669 && pVCpu->hm.s.vmx.cMsrs > 0)
8670 {
8671 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8672 }
8673
8674 /*
8675 * Load the host state bits as we may've been preempted (only happens when
8676 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8677 */
8678 /** @todo Why should hmR0VmxSetupVMRunHandler() changing pfnStartVM have
8679 * any effect to the host state needing to be saved? */
8680 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8681 {
8682 /* This ASSUMES that pfnStartVM has been set up already. */
8683 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8684 AssertRC(rc);
8685 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
8686 }
8687 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8688
8689 /*
8690 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8691 */
8692 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8693 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8694 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8695
8696 /* Store status of the shared guest-host state at the time of VM-entry. */
8697#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8698 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8699 {
8700 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8701 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8702 }
8703 else
8704#endif
8705 {
8706 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8707 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8708 }
8709 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8710
8711 /*
8712 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8713 */
8714 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8715 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8716
8717 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8718 RTCPUID idCurrentCpu = pCpu->idCpu;
8719 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8720 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8721 {
8722 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
8723 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8724 }
8725
8726 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8727 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8728 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8729 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8730
8731 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8732
8733 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8734 to start executing. */
8735
8736 /*
8737 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8738 */
8739 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8740 {
8741 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8742 {
8743 bool fMsrUpdated;
8744 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8745 AssertRC(rc2);
8746 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8747
8748 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8749 &fMsrUpdated);
8750 AssertRC(rc2);
8751 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8752
8753 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8754 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8755 }
8756 else
8757 {
8758 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8759 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8760 }
8761 }
8762
8763#ifdef VBOX_STRICT
8764 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8765 hmR0VmxCheckHostEferMsr(pVCpu);
8766 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8767#endif
8768#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8769 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8770 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8771 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8772#endif
8773}
8774
8775
8776/**
8777 * Performs some essential restoration of state after running guest code in
8778 * VT-x.
8779 *
8780 * @param pVM The cross context VM structure.
8781 * @param pVCpu The cross context virtual CPU structure.
8782 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8783 * out-of-sync. Make sure to update the required fields
8784 * before using them.
8785 * @param pVmxTransient Pointer to the VMX transient structure.
8786 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8787 *
8788 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8789 *
8790 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8791 * unconditionally when it is safe to do so.
8792 */
8793static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8794{
8795 NOREF(pVM);
8796
8797 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8798
8799 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8800 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8801 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8802 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8803 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8804 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8805
8806 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8807 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
8808
8809 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8810 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8811 Assert(!ASMIntAreEnabled());
8812 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8813
8814#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8815 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
8816 {
8817 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8818 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8819 }
8820#endif
8821
8822#if HC_ARCH_BITS == 64
8823 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8824#endif
8825 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8826#ifdef VBOX_STRICT
8827 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8828#endif
8829 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8830 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8831
8832 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8833 uint32_t uExitReason;
8834 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8835 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8836 AssertRC(rc);
8837 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8838 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8839
8840 /* Update the VM-exit history array. */
8841 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
8842
8843 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8844 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8845 {
8846 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8847 pVmxTransient->fVMEntryFailed));
8848 return;
8849 }
8850
8851 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8852 {
8853 /** @todo We can optimize this by only syncing with our force-flags when
8854 * really needed and keeping the VMCS state as it is for most
8855 * VM-exits. */
8856 /* Update the guest interruptibility-state from the VMCS. */
8857 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8858
8859#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8860 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8861 AssertRC(rc);
8862#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8863 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8864 AssertRC(rc);
8865#endif
8866
8867 /*
8868 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8869 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8870 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8871 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8872 */
8873 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8874 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8875 {
8876 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8877 AssertRC(rc);
8878 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8879 }
8880 }
8881}
8882
8883
8884/**
8885 * Runs the guest code using VT-x the normal way.
8886 *
8887 * @returns VBox status code.
8888 * @param pVM The cross context VM structure.
8889 * @param pVCpu The cross context virtual CPU structure.
8890 * @param pCtx Pointer to the guest-CPU context.
8891 *
8892 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8893 */
8894static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8895{
8896 VMXTRANSIENT VmxTransient;
8897 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8898 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8899 uint32_t cLoops = 0;
8900
8901 for (;; cLoops++)
8902 {
8903 Assert(!HMR0SuspendPending());
8904 HMVMX_ASSERT_CPU_SAFE();
8905
8906 /* Preparatory work for running guest code, this may force us to return
8907 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8908 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8909 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8910 if (rcStrict != VINF_SUCCESS)
8911 break;
8912
8913 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8914 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8915 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8916
8917 /* Restore any residual host-state and save any bits shared between host
8918 and guest into the guest-CPU state. Re-enables interrupts! */
8919 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
8920
8921 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8922 if (RT_SUCCESS(rcRun))
8923 { /* very likely */ }
8924 else
8925 {
8926 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8927 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
8928 return rcRun;
8929 }
8930
8931 /* Profile the VM-exit. */
8932 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8934 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8935 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8936 HMVMX_START_EXIT_DISPATCH_PROF();
8937
8938 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8939
8940 /* Handle the VM-exit. */
8941#ifdef HMVMX_USE_FUNCTION_TABLE
8942 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8943#else
8944 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8945#endif
8946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8947 if (rcStrict == VINF_SUCCESS)
8948 {
8949 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
8950 continue; /* likely */
8951 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8952 rcStrict = VINF_EM_RAW_INTERRUPT;
8953 }
8954 break;
8955 }
8956
8957 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8958 return rcStrict;
8959}
8960
8961
8962
8963/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8964 * probes.
8965 *
8966 * The following few functions and associated structure contains the bloat
8967 * necessary for providing detailed debug events and dtrace probes as well as
8968 * reliable host side single stepping. This works on the principle of
8969 * "subclassing" the normal execution loop and workers. We replace the loop
8970 * method completely and override selected helpers to add necessary adjustments
8971 * to their core operation.
8972 *
8973 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8974 * any performance for debug and analysis features.
8975 *
8976 * @{
8977 */
8978
8979typedef struct VMXRUNDBGSTATE
8980{
8981 /** The RIP we started executing at. This is for detecting that we stepped. */
8982 uint64_t uRipStart;
8983 /** The CS we started executing with. */
8984 uint16_t uCsStart;
8985
8986 /** Whether we've actually modified the 1st execution control field. */
8987 bool fModifiedProcCtls : 1;
8988 /** Whether we've actually modified the 2nd execution control field. */
8989 bool fModifiedProcCtls2 : 1;
8990 /** Whether we've actually modified the exception bitmap. */
8991 bool fModifiedXcptBitmap : 1;
8992
8993 /** We desire the modified the CR0 mask to be cleared. */
8994 bool fClearCr0Mask : 1;
8995 /** We desire the modified the CR4 mask to be cleared. */
8996 bool fClearCr4Mask : 1;
8997 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8998 uint32_t fCpe1Extra;
8999 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9000 uint32_t fCpe1Unwanted;
9001 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9002 uint32_t fCpe2Extra;
9003 /** Extra stuff we need in */
9004 uint32_t bmXcptExtra;
9005 /** The sequence number of the Dtrace provider settings the state was
9006 * configured against. */
9007 uint32_t uDtraceSettingsSeqNo;
9008 /** Exits to check (one bit per exit). */
9009 uint32_t bmExitsToCheck[3];
9010
9011 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9012 uint32_t fProcCtlsInitial;
9013 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9014 uint32_t fProcCtls2Initial;
9015 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9016 uint32_t bmXcptInitial;
9017} VMXRUNDBGSTATE;
9018AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9019typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9020
9021
9022/**
9023 * Initializes the VMXRUNDBGSTATE structure.
9024 *
9025 * @param pVCpu The cross context virtual CPU structure of the
9026 * calling EMT.
9027 * @param pCtx The CPU register context to go with @a pVCpu.
9028 * @param pDbgState The structure to initialize.
9029 */
9030DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9031{
9032 pDbgState->uRipStart = pCtx->rip;
9033 pDbgState->uCsStart = pCtx->cs.Sel;
9034
9035 pDbgState->fModifiedProcCtls = false;
9036 pDbgState->fModifiedProcCtls2 = false;
9037 pDbgState->fModifiedXcptBitmap = false;
9038 pDbgState->fClearCr0Mask = false;
9039 pDbgState->fClearCr4Mask = false;
9040 pDbgState->fCpe1Extra = 0;
9041 pDbgState->fCpe1Unwanted = 0;
9042 pDbgState->fCpe2Extra = 0;
9043 pDbgState->bmXcptExtra = 0;
9044 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9045 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9046 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9047}
9048
9049
9050/**
9051 * Updates the VMSC fields with changes requested by @a pDbgState.
9052 *
9053 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9054 * immediately before executing guest code, i.e. when interrupts are disabled.
9055 * We don't check status codes here as we cannot easily assert or return in the
9056 * latter case.
9057 *
9058 * @param pVCpu The cross context virtual CPU structure.
9059 * @param pDbgState The debug state.
9060 */
9061DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9062{
9063 /*
9064 * Ensure desired flags in VMCS control fields are set.
9065 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9066 *
9067 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9068 * there should be no stale data in pCtx at this point.
9069 */
9070 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9071 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9072 {
9073 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9074 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9075 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9076 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9077 pDbgState->fModifiedProcCtls = true;
9078 }
9079
9080 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9081 {
9082 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9083 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9084 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9085 pDbgState->fModifiedProcCtls2 = true;
9086 }
9087
9088 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9089 {
9090 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9091 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9092 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9093 pDbgState->fModifiedXcptBitmap = true;
9094 }
9095
9096 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9097 {
9098 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9099 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9100 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9101 }
9102
9103 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9104 {
9105 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9106 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9107 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9108 }
9109}
9110
9111
9112DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9113{
9114 /*
9115 * Restore exit control settings as we may not reenter this function the
9116 * next time around.
9117 */
9118 /* We reload the initial value, trigger what we can of recalculations the
9119 next time around. From the looks of things, that's all that's required atm. */
9120 if (pDbgState->fModifiedProcCtls)
9121 {
9122 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9123 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9124 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9125 AssertRCReturn(rc2, rc2);
9126 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9127 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_DEBUG);
9128 }
9129
9130 /* We're currently the only ones messing with this one, so just restore the
9131 cached value and reload the field. */
9132 if ( pDbgState->fModifiedProcCtls2
9133 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9134 {
9135 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9136 AssertRCReturn(rc2, rc2);
9137 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9138 }
9139
9140 /* If we've modified the exception bitmap, we restore it and trigger
9141 reloading and partial recalculation the next time around. */
9142 if (pDbgState->fModifiedXcptBitmap)
9143 {
9144 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9145 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_XCPT_INTERCEPTS | HM_CHANGED_GUEST_CR0);
9146 }
9147
9148 /* We assume hmR0VmxLoadSharedCR0 will recalculate and load the CR0 mask. */
9149 if (pDbgState->fClearCr0Mask)
9150 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9151
9152 /* We assume hmR0VmxLoadGuestCR3AndCR4 will recalculate and load the CR4 mask. */
9153 if (pDbgState->fClearCr4Mask)
9154 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9155
9156 return rcStrict;
9157}
9158
9159
9160/**
9161 * Configures VM-exit controls for current DBGF and DTrace settings.
9162 *
9163 * This updates @a pDbgState and the VMCS execution control fields to reflect
9164 * the necessary exits demanded by DBGF and DTrace.
9165 *
9166 * @param pVM The cross context VM structure.
9167 * @param pVCpu The cross context virtual CPU structure.
9168 * @param pCtx Pointer to the guest-CPU context.
9169 * @param pDbgState The debug state.
9170 * @param pVmxTransient Pointer to the VMX transient structure. May update
9171 * fUpdateTscOffsettingAndPreemptTimer.
9172 */
9173static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9174 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9175{
9176 /*
9177 * Take down the dtrace serial number so we can spot changes.
9178 */
9179 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9180 ASMCompilerBarrier();
9181
9182 /*
9183 * We'll rebuild most of the middle block of data members (holding the
9184 * current settings) as we go along here, so start by clearing it all.
9185 */
9186 pDbgState->bmXcptExtra = 0;
9187 pDbgState->fCpe1Extra = 0;
9188 pDbgState->fCpe1Unwanted = 0;
9189 pDbgState->fCpe2Extra = 0;
9190 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9191 pDbgState->bmExitsToCheck[i] = 0;
9192
9193 /*
9194 * Software interrupts (INT XXh) - no idea how to trigger these...
9195 */
9196 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9197 || VBOXVMM_INT_SOFTWARE_ENABLED())
9198 {
9199 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9200 }
9201
9202 /*
9203 * Exception bitmap and XCPT events+probes.
9204 */
9205 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9206 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9207 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9208
9209 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9210 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9211 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9212 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9213 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9214 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9215 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9216 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9217 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9218 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9219 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9220 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9221 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9222 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9223 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9224 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9225 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9226 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9227
9228 if (pDbgState->bmXcptExtra)
9229 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9230
9231 /*
9232 * Process events and probes for VM exits, making sure we get the wanted exits.
9233 *
9234 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9235 * So, when adding/changing/removing please don't forget to update it.
9236 *
9237 * Some of the macros are picking up local variables to save horizontal space,
9238 * (being able to see it in a table is the lesser evil here).
9239 */
9240#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9241 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9242 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9243#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9244 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9245 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9246 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9247 } else do { } while (0)
9248#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9249 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9250 { \
9251 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9252 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9253 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9254 } else do { } while (0)
9255#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9256 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9257 { \
9258 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9259 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9260 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9261 } else do { } while (0)
9262#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9263 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9264 { \
9265 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9266 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9267 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9268 } else do { } while (0)
9269
9270 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9271 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9272 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9273 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9274 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9275
9276 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9278 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9280 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9282 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9284 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9285 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9286 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9288 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9292 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9296 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9298 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9302 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9305 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9308 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9309 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9310 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9311 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9312
9313 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9314 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9315 {
9316 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9317 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9318 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9319 AssertRC(rc2);
9320
9321#if 0 /** @todo fix me */
9322 pDbgState->fClearCr0Mask = true;
9323 pDbgState->fClearCr4Mask = true;
9324#endif
9325 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9326 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9327 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9328 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9329 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9330 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9331 require clearing here and in the loop if we start using it. */
9332 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9333 }
9334 else
9335 {
9336 if (pDbgState->fClearCr0Mask)
9337 {
9338 pDbgState->fClearCr0Mask = false;
9339 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9340 }
9341 if (pDbgState->fClearCr4Mask)
9342 {
9343 pDbgState->fClearCr4Mask = false;
9344 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9345 }
9346 }
9347 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9348 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9349
9350 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9351 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9352 {
9353 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9354 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9355 }
9356 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9357 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9358
9359 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9360 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9361 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9362 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9363 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9364 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9365 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9366 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9367#if 0 /** @todo too slow, fix handler. */
9368 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9369#endif
9370 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9371
9372 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9373 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9374 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9375 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9376 {
9377 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9378 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9379 }
9380 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9381 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9382 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9383 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9384
9385 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9386 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9387 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9388 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9389 {
9390 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9391 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9392 }
9393 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9394 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9395 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9396 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9397
9398 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9399 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9400 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9401 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9402 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9403 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9404 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9405 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9406 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9407 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9408 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9409 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9410 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9411 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9412 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9413 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9414 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9415 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9416 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9417 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9418 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9419 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9420
9421#undef IS_EITHER_ENABLED
9422#undef SET_ONLY_XBM_IF_EITHER_EN
9423#undef SET_CPE1_XBM_IF_EITHER_EN
9424#undef SET_CPEU_XBM_IF_EITHER_EN
9425#undef SET_CPE2_XBM_IF_EITHER_EN
9426
9427 /*
9428 * Sanitize the control stuff.
9429 */
9430 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9431 if (pDbgState->fCpe2Extra)
9432 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9433 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9434 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9435 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9436 {
9437 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9438 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9439 }
9440
9441 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9442 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9443 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9444 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9445}
9446
9447
9448/**
9449 * Fires off DBGF events and dtrace probes for an exit, when it's appropriate.
9450 *
9451 * The caller has checked exit against the VMXRUNDBGSTATE::bmExitsToCheck
9452 * bitmap. The caller has checked for NMIs already, so we don't have to do that
9453 * either.
9454 *
9455 * @returns Strict VBox status code (i.e. informational status codes too).
9456 * @param pVM The cross context VM structure.
9457 * @param pVCpu The cross context virtual CPU structure.
9458 * @param pMixedCtx Pointer to the guest-CPU context.
9459 * @param pVmxTransient Pointer to the VMX-transient structure.
9460 * @param uExitReason The VM-exit reason.
9461 *
9462 * @remarks The name of this function is displayed by dtrace, so keep it short
9463 * and to the point. No longer than 33 chars long, please.
9464 */
9465static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9466 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9467{
9468 /*
9469 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9470 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9471 *
9472 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9473 * does. Must add/change/remove both places. Same ordering, please.
9474 *
9475 * Added/removed events must also be reflected in the next section
9476 * where we dispatch dtrace events.
9477 */
9478 bool fDtrace1 = false;
9479 bool fDtrace2 = false;
9480 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9481 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9482 uint32_t uEventArg = 0;
9483#define SET_EXIT(a_EventSubName) \
9484 do { \
9485 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9486 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9487 } while (0)
9488#define SET_BOTH(a_EventSubName) \
9489 do { \
9490 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9491 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9492 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9493 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9494 } while (0)
9495 switch (uExitReason)
9496 {
9497 case VMX_EXIT_MTF:
9498 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9499
9500 case VMX_EXIT_XCPT_OR_NMI:
9501 {
9502 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9503 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9504 {
9505 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9506 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9507 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9508 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9509 {
9510 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9511 {
9512 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9513 uEventArg = pVmxTransient->uExitIntErrorCode;
9514 }
9515 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9516 switch (enmEvent1)
9517 {
9518 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9519 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9520 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9521 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9522 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9523 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9524 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9525 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9526 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9527 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9528 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9529 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9530 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9531 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9532 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9533 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9534 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9535 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9536 default: break;
9537 }
9538 }
9539 else
9540 AssertFailed();
9541 break;
9542
9543 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9544 uEventArg = idxVector;
9545 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9546 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9547 break;
9548 }
9549 break;
9550 }
9551
9552 case VMX_EXIT_TRIPLE_FAULT:
9553 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9554 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9555 break;
9556 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9557 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9558 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9559 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9560 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9561
9562 /* Instruction specific VM-exits: */
9563 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9564 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9565 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9566 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9567 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9568 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9569 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9570 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9571 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9572 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9573 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9574 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9575 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9576 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9577 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9578 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9579 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9580 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9581 case VMX_EXIT_MOV_CRX:
9582 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9583/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
9584* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
9585 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
9586 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
9587 SET_BOTH(CRX_READ);
9588 else
9589 SET_BOTH(CRX_WRITE);
9590 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
9591 break;
9592 case VMX_EXIT_MOV_DRX:
9593 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9594 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
9595 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
9596 SET_BOTH(DRX_READ);
9597 else
9598 SET_BOTH(DRX_WRITE);
9599 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
9600 break;
9601 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9602 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9603 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9604 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9605 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9606 case VMX_EXIT_XDTR_ACCESS:
9607 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9608 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9609 {
9610 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9611 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9612 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9613 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9614 }
9615 break;
9616
9617 case VMX_EXIT_TR_ACCESS:
9618 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9619 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9620 {
9621 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9622 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9623 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9624 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9625 }
9626 break;
9627
9628 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9629 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9630 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9631 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9632 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9633 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9634 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9635 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9636 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9637 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9638 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9639
9640 /* Events that aren't relevant at this point. */
9641 case VMX_EXIT_EXT_INT:
9642 case VMX_EXIT_INT_WINDOW:
9643 case VMX_EXIT_NMI_WINDOW:
9644 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9645 case VMX_EXIT_PREEMPT_TIMER:
9646 case VMX_EXIT_IO_INSTR:
9647 break;
9648
9649 /* Errors and unexpected events. */
9650 case VMX_EXIT_INIT_SIGNAL:
9651 case VMX_EXIT_SIPI:
9652 case VMX_EXIT_IO_SMI:
9653 case VMX_EXIT_SMI:
9654 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9655 case VMX_EXIT_ERR_MSR_LOAD:
9656 case VMX_EXIT_ERR_MACHINE_CHECK:
9657 break;
9658
9659 default:
9660 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9661 break;
9662 }
9663#undef SET_BOTH
9664#undef SET_EXIT
9665
9666 /*
9667 * Dtrace tracepoints go first. We do them here at once so we don't
9668 * have to copy the guest state saving and stuff a few dozen times.
9669 * Down side is that we've got to repeat the switch, though this time
9670 * we use enmEvent since the probes are a subset of what DBGF does.
9671 */
9672 if (fDtrace1 || fDtrace2)
9673 {
9674 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9675 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9676 switch (enmEvent1)
9677 {
9678 /** @todo consider which extra parameters would be helpful for each probe. */
9679 case DBGFEVENT_END: break;
9680 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9681 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9682 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9683 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9684 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9685 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9686 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9688 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9689 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9690 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9691 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9692 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9693 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9694 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9695 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9696 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9697 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9698 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9699 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9700 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9701 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9702 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9703 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9704 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9705 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9706 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9707 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9708 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9709 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9710 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9711 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9712 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9713 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9714 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9715 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9716 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9717 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9718 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9719 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9720 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9721 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9722 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9723 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9724 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9725 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9726 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9727 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9728 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9729 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9730 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9731 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9732 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9733 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9734 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9735 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9736 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9737 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9738 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9739 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9740 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9741 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9742 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9743 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9744 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9745 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9746 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9747 }
9748 switch (enmEvent2)
9749 {
9750 /** @todo consider which extra parameters would be helpful for each probe. */
9751 case DBGFEVENT_END: break;
9752 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9753 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9754 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9755 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9756 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9757 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9758 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9759 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9760 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9761 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9762 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9763 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9764 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9765 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9766 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9767 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9768 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9769 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9770 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9771 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9772 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9773 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9774 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9775 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9776 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9777 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9778 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9779 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9780 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9781 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9782 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9783 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9784 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9785 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9786 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9787 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9788 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9789 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9790 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9791 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9792 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9793 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9794 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9795 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9796 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9797 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9798 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9799 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9800 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9801 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9802 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9803 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9804 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9805 }
9806 }
9807
9808 /*
9809 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9810 * the DBGF call will do a full check).
9811 *
9812 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9813 * Note! If we have to events, we prioritize the first, i.e. the instruction
9814 * one, in order to avoid event nesting.
9815 */
9816 if ( enmEvent1 != DBGFEVENT_END
9817 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9818 {
9819 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9820 if (rcStrict != VINF_SUCCESS)
9821 return rcStrict;
9822 }
9823 else if ( enmEvent2 != DBGFEVENT_END
9824 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9825 {
9826 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9827 if (rcStrict != VINF_SUCCESS)
9828 return rcStrict;
9829 }
9830
9831 return VINF_SUCCESS;
9832}
9833
9834
9835/**
9836 * Single-stepping VM-exit filtering.
9837 *
9838 * This is preprocessing the exits and deciding whether we've gotten far enough
9839 * to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit handling is
9840 * performed.
9841 *
9842 * @returns Strict VBox status code (i.e. informational status codes too).
9843 * @param pVM The cross context VM structure.
9844 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9845 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9846 * out-of-sync. Make sure to update the required
9847 * fields before using them.
9848 * @param pVmxTransient Pointer to the VMX-transient structure.
9849 * @param uExitReason The VM-exit reason.
9850 * @param pDbgState The debug state.
9851 */
9852DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9853 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9854{
9855 /*
9856 * Expensive (saves context) generic dtrace exit probe.
9857 */
9858 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9859 { /* more likely */ }
9860 else
9861 {
9862 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9863 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9864 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9865 }
9866
9867 /*
9868 * Check for host NMI, just to get that out of the way.
9869 */
9870 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9871 { /* normally likely */ }
9872 else
9873 {
9874 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9875 AssertRCReturn(rc2, rc2);
9876 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9877 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9878 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9879 }
9880
9881 /*
9882 * Check for single stepping event if we're stepping.
9883 */
9884 if (pVCpu->hm.s.fSingleInstruction)
9885 {
9886 switch (uExitReason)
9887 {
9888 case VMX_EXIT_MTF:
9889 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9890
9891 /* Various events: */
9892 case VMX_EXIT_XCPT_OR_NMI:
9893 case VMX_EXIT_EXT_INT:
9894 case VMX_EXIT_TRIPLE_FAULT:
9895 case VMX_EXIT_INT_WINDOW:
9896 case VMX_EXIT_NMI_WINDOW:
9897 case VMX_EXIT_TASK_SWITCH:
9898 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9899 case VMX_EXIT_APIC_ACCESS:
9900 case VMX_EXIT_EPT_VIOLATION:
9901 case VMX_EXIT_EPT_MISCONFIG:
9902 case VMX_EXIT_PREEMPT_TIMER:
9903
9904 /* Instruction specific VM-exits: */
9905 case VMX_EXIT_CPUID:
9906 case VMX_EXIT_GETSEC:
9907 case VMX_EXIT_HLT:
9908 case VMX_EXIT_INVD:
9909 case VMX_EXIT_INVLPG:
9910 case VMX_EXIT_RDPMC:
9911 case VMX_EXIT_RDTSC:
9912 case VMX_EXIT_RSM:
9913 case VMX_EXIT_VMCALL:
9914 case VMX_EXIT_VMCLEAR:
9915 case VMX_EXIT_VMLAUNCH:
9916 case VMX_EXIT_VMPTRLD:
9917 case VMX_EXIT_VMPTRST:
9918 case VMX_EXIT_VMREAD:
9919 case VMX_EXIT_VMRESUME:
9920 case VMX_EXIT_VMWRITE:
9921 case VMX_EXIT_VMXOFF:
9922 case VMX_EXIT_VMXON:
9923 case VMX_EXIT_MOV_CRX:
9924 case VMX_EXIT_MOV_DRX:
9925 case VMX_EXIT_IO_INSTR:
9926 case VMX_EXIT_RDMSR:
9927 case VMX_EXIT_WRMSR:
9928 case VMX_EXIT_MWAIT:
9929 case VMX_EXIT_MONITOR:
9930 case VMX_EXIT_PAUSE:
9931 case VMX_EXIT_XDTR_ACCESS:
9932 case VMX_EXIT_TR_ACCESS:
9933 case VMX_EXIT_INVEPT:
9934 case VMX_EXIT_RDTSCP:
9935 case VMX_EXIT_INVVPID:
9936 case VMX_EXIT_WBINVD:
9937 case VMX_EXIT_XSETBV:
9938 case VMX_EXIT_RDRAND:
9939 case VMX_EXIT_INVPCID:
9940 case VMX_EXIT_VMFUNC:
9941 case VMX_EXIT_RDSEED:
9942 case VMX_EXIT_XSAVES:
9943 case VMX_EXIT_XRSTORS:
9944 {
9945 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9946 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9947 AssertRCReturn(rc2, rc2);
9948 if ( pMixedCtx->rip != pDbgState->uRipStart
9949 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9950 return VINF_EM_DBG_STEPPED;
9951 break;
9952 }
9953
9954 /* Errors and unexpected events: */
9955 case VMX_EXIT_INIT_SIGNAL:
9956 case VMX_EXIT_SIPI:
9957 case VMX_EXIT_IO_SMI:
9958 case VMX_EXIT_SMI:
9959 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9960 case VMX_EXIT_ERR_MSR_LOAD:
9961 case VMX_EXIT_ERR_MACHINE_CHECK:
9962 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9963 break;
9964
9965 default:
9966 AssertMsgFailed(("Unexpected exit=%#x\n", uExitReason));
9967 break;
9968 }
9969 }
9970
9971 /*
9972 * Check for debugger event breakpoints and dtrace probes.
9973 */
9974 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9975 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9976 {
9977 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9978 if (rcStrict != VINF_SUCCESS)
9979 return rcStrict;
9980 }
9981
9982 /*
9983 * Normal processing.
9984 */
9985#ifdef HMVMX_USE_FUNCTION_TABLE
9986 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9987#else
9988 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9989#endif
9990}
9991
9992
9993/**
9994 * Single steps guest code using VT-x.
9995 *
9996 * @returns Strict VBox status code (i.e. informational status codes too).
9997 * @param pVM The cross context VM structure.
9998 * @param pVCpu The cross context virtual CPU structure.
9999 * @param pCtx Pointer to the guest-CPU context.
10000 *
10001 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10002 */
10003static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10004{
10005 VMXTRANSIENT VmxTransient;
10006 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10007
10008 /* Set HMCPU indicators. */
10009 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10010 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10011 pVCpu->hm.s.fDebugWantRdTscExit = false;
10012 pVCpu->hm.s.fUsingDebugLoop = true;
10013
10014 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10015 VMXRUNDBGSTATE DbgState;
10016 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10017 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10018
10019 /*
10020 * The loop.
10021 */
10022 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10023 for (uint32_t cLoops = 0; ; cLoops++)
10024 {
10025 Assert(!HMR0SuspendPending());
10026 HMVMX_ASSERT_CPU_SAFE();
10027 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10028
10029 /*
10030 * Preparatory work for running guest code, this may force us to return
10031 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10032 */
10033 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10034 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10035 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10036 if (rcStrict != VINF_SUCCESS)
10037 break;
10038
10039 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10040 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10041
10042 /*
10043 * Now we can run the guest code.
10044 */
10045 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10046
10047 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10048
10049 /*
10050 * Restore any residual host-state and save any bits shared between host
10051 * and guest into the guest-CPU state. Re-enables interrupts!
10052 */
10053 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, VBOXSTRICTRC_TODO(rcStrict));
10054
10055 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10056 if (RT_SUCCESS(rcRun))
10057 { /* very likely */ }
10058 else
10059 {
10060 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10061 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10062 return rcRun;
10063 }
10064
10065 /* Profile the VM-exit. */
10066 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10067 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10068 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10069 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10070 HMVMX_START_EXIT_DISPATCH_PROF();
10071
10072 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10073
10074 /*
10075 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10076 */
10077 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10078 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10079 if (rcStrict != VINF_SUCCESS)
10080 break;
10081 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10082 {
10083 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10084 rcStrict = VINF_EM_RAW_INTERRUPT;
10085 break;
10086 }
10087
10088 /*
10089 * Stepping: Did the RIP change, if so, consider it a single step.
10090 * Otherwise, make sure one of the TFs gets set.
10091 */
10092 if (fStepping)
10093 {
10094 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10095 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10096 AssertRCReturn(rc2, rc2);
10097 if ( pCtx->rip != DbgState.uRipStart
10098 || pCtx->cs.Sel != DbgState.uCsStart)
10099 {
10100 rcStrict = VINF_EM_DBG_STEPPED;
10101 break;
10102 }
10103 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10104 }
10105
10106 /*
10107 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10108 */
10109 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10110 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10111 }
10112
10113 /*
10114 * Clear the X86_EFL_TF if necessary.
10115 */
10116 if (pVCpu->hm.s.fClearTrapFlag)
10117 {
10118 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10119 AssertRCReturn(rc2, rc2);
10120 pVCpu->hm.s.fClearTrapFlag = false;
10121 pCtx->eflags.Bits.u1TF = 0;
10122 }
10123 /** @todo there seems to be issues with the resume flag when the monitor trap
10124 * flag is pending without being used. Seen early in bios init when
10125 * accessing APIC page in protected mode. */
10126
10127 /*
10128 * Restore VM-exit control settings as we may not reenter this function the
10129 * next time around.
10130 */
10131 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10132
10133 /* Restore HMCPU indicators. */
10134 pVCpu->hm.s.fUsingDebugLoop = false;
10135 pVCpu->hm.s.fDebugWantRdTscExit = false;
10136 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10137
10138 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10139 return rcStrict;
10140}
10141
10142
10143/** @} */
10144
10145
10146/**
10147 * Checks if any expensive dtrace probes are enabled and we should go to the
10148 * debug loop.
10149 *
10150 * @returns true if we should use debug loop, false if not.
10151 */
10152static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10153{
10154 /* It's probably faster to OR the raw 32-bit counter variables together.
10155 Since the variables are in an array and the probes are next to one
10156 another (more or less), we have good locality. So, better read
10157 eight-nine cache lines ever time and only have one conditional, than
10158 128+ conditionals, right? */
10159 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10160 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10161 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10162 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10163 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10164 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10165 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10166 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10167 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10168 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10169 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10170 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10171 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10172 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10173 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10174 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10175 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10176 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10177 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10178 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10179 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10180 ) != 0
10181 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10182 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10183 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10184 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10185 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10186 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10187 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10188 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10189 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10190 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10191 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10192 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10193 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10194 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10195 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10196 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10197 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10198 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10199 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10200 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10201 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10202 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10203 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10204 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10205 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10206 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10207 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10208 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10209 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10210 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10211 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10212 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10213 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10214 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10215 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10216 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10217 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10218 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10219 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10220 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10221 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10222 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10223 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10224 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10225 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10226 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10227 ) != 0
10228 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10229 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10230 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10231 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10232 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10233 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10234 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10235 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10236 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10237 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10238 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10239 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10240 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10241 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10242 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10243 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10244 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10245 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10246 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10247 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10248 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10249 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10250 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10251 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10252 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10253 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10254 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10255 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10256 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10257 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10258 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10259 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10260 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10261 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10262 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10263 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10264 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10265 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10266 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10267 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10268 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10269 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10270 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10271 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10272 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10273 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10274 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10275 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10276 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10277 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10278 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10279 ) != 0;
10280}
10281
10282
10283/**
10284 * Runs the guest code using VT-x.
10285 *
10286 * @returns Strict VBox status code (i.e. informational status codes too).
10287 * @param pVM The cross context VM structure.
10288 * @param pVCpu The cross context virtual CPU structure.
10289 * @param pCtx Pointer to the guest-CPU context.
10290 */
10291VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10292{
10293 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10294 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10295 HMVMX_ASSERT_PREEMPT_SAFE();
10296
10297 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10298
10299 VBOXSTRICTRC rcStrict;
10300 if ( !pVCpu->hm.s.fUseDebugLoop
10301 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10302 && !DBGFIsStepping(pVCpu) )
10303 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10304 else
10305 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10306
10307 if (rcStrict == VERR_EM_INTERPRETER)
10308 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10309 else if (rcStrict == VINF_EM_RESET)
10310 rcStrict = VINF_EM_TRIPLE_FAULT;
10311
10312 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10313 if (RT_FAILURE(rc2))
10314 {
10315 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10316 rcStrict = rc2;
10317 }
10318 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10319 return rcStrict;
10320}
10321
10322
10323#ifndef HMVMX_USE_FUNCTION_TABLE
10324DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10325{
10326# ifdef DEBUG_ramshankar
10327# define RETURN_EXIT_CALL(a_CallExpr) \
10328 do { \
10329 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10330 VBOXSTRICTRC rcStrict = a_CallExpr; \
10331 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10332 return rcStrict; \
10333 } while (0)
10334# else
10335# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10336# endif
10337 switch (rcReason)
10338 {
10339 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10340 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10341 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10342 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10343 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10344 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10345 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10346 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10347 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10348 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10349 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10350 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10351 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10352 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10353 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10354 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10355 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10356 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10357 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10358 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10359 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10360 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10361 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10362 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10363 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10364 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10365 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10366 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10367 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10368 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10369 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10370 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10371 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10372 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10373
10374 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10375 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10376 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10377 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10378 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10379 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10380 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10381 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10382 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10383
10384 case VMX_EXIT_VMCLEAR:
10385 case VMX_EXIT_VMLAUNCH:
10386 case VMX_EXIT_VMPTRLD:
10387 case VMX_EXIT_VMPTRST:
10388 case VMX_EXIT_VMREAD:
10389 case VMX_EXIT_VMRESUME:
10390 case VMX_EXIT_VMWRITE:
10391 case VMX_EXIT_VMXOFF:
10392 case VMX_EXIT_VMXON:
10393 case VMX_EXIT_INVEPT:
10394 case VMX_EXIT_INVVPID:
10395 case VMX_EXIT_VMFUNC:
10396 case VMX_EXIT_XSAVES:
10397 case VMX_EXIT_XRSTORS:
10398 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10399 case VMX_EXIT_RESERVED_60:
10400 case VMX_EXIT_RDSEED: /* only spurious exits, so undefined */
10401 case VMX_EXIT_RESERVED_62:
10402 default:
10403 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10404 }
10405#undef RETURN_EXIT_CALL
10406}
10407#endif /* !HMVMX_USE_FUNCTION_TABLE */
10408
10409
10410#ifdef VBOX_STRICT
10411/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10412# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10413 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10414
10415# define HMVMX_ASSERT_PREEMPT_CPUID() \
10416 do { \
10417 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10418 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10419 } while (0)
10420
10421# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10422 do { \
10423 AssertPtr(pVCpu); \
10424 AssertPtr(pMixedCtx); \
10425 AssertPtr(pVmxTransient); \
10426 Assert(pVmxTransient->fVMEntryFailed == false); \
10427 Assert(ASMIntAreEnabled()); \
10428 HMVMX_ASSERT_PREEMPT_SAFE(); \
10429 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10430 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10431 HMVMX_ASSERT_PREEMPT_SAFE(); \
10432 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10433 HMVMX_ASSERT_PREEMPT_CPUID(); \
10434 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10435 } while (0)
10436
10437# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10438 do { \
10439 Log4Func(("\n")); \
10440 } while (0)
10441#else /* nonstrict builds: */
10442# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10443 do { \
10444 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10445 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10446 } while (0)
10447# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10448#endif
10449
10450
10451/**
10452 * Advances the guest RIP by the specified number of bytes.
10453 *
10454 * @param pVCpu The cross context virtual CPU structure.
10455 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10456 * out-of-sync. Make sure to update the required fields
10457 * before using them.
10458 * @param cbInstr Number of bytes to advance the RIP by.
10459 *
10460 * @remarks No-long-jump zone!!!
10461 */
10462DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10463{
10464 /* Advance the RIP. */
10465 pMixedCtx->rip += cbInstr;
10466 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10467
10468 /* Update interrupt inhibition. */
10469 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10470 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10471 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10472}
10473
10474
10475/**
10476 * Advances the guest RIP after reading it from the VMCS.
10477 *
10478 * @returns VBox status code, no informational status codes.
10479 * @param pVCpu The cross context virtual CPU structure.
10480 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10481 * out-of-sync. Make sure to update the required fields
10482 * before using them.
10483 * @param pVmxTransient Pointer to the VMX transient structure.
10484 *
10485 * @remarks No-long-jump zone!!!
10486 */
10487static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10488{
10489 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10490 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10491 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10492 AssertRCReturn(rc, rc);
10493
10494 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10495
10496 /*
10497 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10498 * pending debug exception field as it takes care of priority of events.
10499 *
10500 * See Intel spec. 32.2.1 "Debug Exceptions".
10501 */
10502 if ( !pVCpu->hm.s.fSingleInstruction
10503 && pMixedCtx->eflags.Bits.u1TF)
10504 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10505
10506 return VINF_SUCCESS;
10507}
10508
10509
10510/**
10511 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10512 * and update error record fields accordingly.
10513 *
10514 * @return VMX_IGS_* return codes.
10515 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10516 * wrong with the guest state.
10517 *
10518 * @param pVM The cross context VM structure.
10519 * @param pVCpu The cross context virtual CPU structure.
10520 * @param pCtx Pointer to the guest-CPU state.
10521 *
10522 * @remarks This function assumes our cache of the VMCS controls
10523 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10524 */
10525static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10526{
10527#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10528#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10529 uError = (err); \
10530 break; \
10531 } else do { } while (0)
10532
10533 int rc;
10534 uint32_t uError = VMX_IGS_ERROR;
10535 uint32_t u32Val;
10536 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10537
10538 do
10539 {
10540 /*
10541 * CR0.
10542 */
10543 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10544 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10545 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10546 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10547 if (fUnrestrictedGuest)
10548 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10549
10550 uint32_t u32GuestCR0;
10551 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10552 AssertRCBreak(rc);
10553 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10554 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10555 if ( !fUnrestrictedGuest
10556 && (u32GuestCR0 & X86_CR0_PG)
10557 && !(u32GuestCR0 & X86_CR0_PE))
10558 {
10559 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10560 }
10561
10562 /*
10563 * CR4.
10564 */
10565 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10566 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10567
10568 uint32_t u32GuestCR4;
10569 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
10570 AssertRCBreak(rc);
10571 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
10572 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
10573
10574 /*
10575 * IA32_DEBUGCTL MSR.
10576 */
10577 uint64_t u64Val;
10578 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10579 AssertRCBreak(rc);
10580 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10581 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10582 {
10583 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10584 }
10585 uint64_t u64DebugCtlMsr = u64Val;
10586
10587#ifdef VBOX_STRICT
10588 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10589 AssertRCBreak(rc);
10590 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10591#endif
10592 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10593
10594 /*
10595 * RIP and RFLAGS.
10596 */
10597 uint32_t u32Eflags;
10598#if HC_ARCH_BITS == 64
10599 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10600 AssertRCBreak(rc);
10601 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10602 if ( !fLongModeGuest
10603 || !pCtx->cs.Attr.n.u1Long)
10604 {
10605 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10606 }
10607 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10608 * must be identical if the "IA-32e mode guest" VM-entry
10609 * control is 1 and CS.L is 1. No check applies if the
10610 * CPU supports 64 linear-address bits. */
10611
10612 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10613 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10614 AssertRCBreak(rc);
10615 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10616 VMX_IGS_RFLAGS_RESERVED);
10617 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10618 u32Eflags = u64Val;
10619#else
10620 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10621 AssertRCBreak(rc);
10622 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10623 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10624#endif
10625
10626 if ( fLongModeGuest
10627 || ( fUnrestrictedGuest
10628 && !(u32GuestCR0 & X86_CR0_PE)))
10629 {
10630 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10631 }
10632
10633 uint32_t u32EntryInfo;
10634 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10635 AssertRCBreak(rc);
10636 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10637 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10638 {
10639 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10640 }
10641
10642 /*
10643 * 64-bit checks.
10644 */
10645#if HC_ARCH_BITS == 64
10646 if (fLongModeGuest)
10647 {
10648 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10649 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10650 }
10651
10652 if ( !fLongModeGuest
10653 && (u32GuestCR4 & X86_CR4_PCIDE))
10654 {
10655 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10656 }
10657
10658 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10659 * 51:32 beyond the processor's physical-address width are 0. */
10660
10661 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10662 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10663 {
10664 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10665 }
10666
10667 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10668 AssertRCBreak(rc);
10669 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10670
10671 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10672 AssertRCBreak(rc);
10673 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10674#endif
10675
10676 /*
10677 * PERF_GLOBAL MSR.
10678 */
10679 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10680 {
10681 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10682 AssertRCBreak(rc);
10683 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10684 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10685 }
10686
10687 /*
10688 * PAT MSR.
10689 */
10690 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10691 {
10692 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10693 AssertRCBreak(rc);
10694 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10695 for (unsigned i = 0; i < 8; i++)
10696 {
10697 uint8_t u8Val = (u64Val & 0xff);
10698 if ( u8Val != 0 /* UC */
10699 && u8Val != 1 /* WC */
10700 && u8Val != 4 /* WT */
10701 && u8Val != 5 /* WP */
10702 && u8Val != 6 /* WB */
10703 && u8Val != 7 /* UC- */)
10704 {
10705 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10706 }
10707 u64Val >>= 8;
10708 }
10709 }
10710
10711 /*
10712 * EFER MSR.
10713 */
10714 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10715 {
10716 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10717 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10718 AssertRCBreak(rc);
10719 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10720 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10721 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10722 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10723 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10724 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10725 || !(u32GuestCR0 & X86_CR0_PG)
10726 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10727 VMX_IGS_EFER_LMA_LME_MISMATCH);
10728 }
10729
10730 /*
10731 * Segment registers.
10732 */
10733 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10734 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10735 if (!(u32Eflags & X86_EFL_VM))
10736 {
10737 /* CS */
10738 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10739 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10740 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10741 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10742 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10743 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10744 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10745 /* CS cannot be loaded with NULL in protected mode. */
10746 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10747 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10748 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10749 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10750 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10751 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10752 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10753 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10754 else
10755 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10756
10757 /* SS */
10758 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10759 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10760 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10761 if ( !(pCtx->cr0 & X86_CR0_PE)
10762 || pCtx->cs.Attr.n.u4Type == 3)
10763 {
10764 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10765 }
10766 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10767 {
10768 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10769 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10770 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10771 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10772 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10773 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10774 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10775 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10776 }
10777
10778 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10779 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10780 {
10781 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10782 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10783 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10784 || pCtx->ds.Attr.n.u4Type > 11
10785 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10786 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10787 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10788 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10789 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10790 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10791 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10792 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10793 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10794 }
10795 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10796 {
10797 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10798 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10799 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10800 || pCtx->es.Attr.n.u4Type > 11
10801 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10802 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10803 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10804 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10805 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10806 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10807 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10808 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10809 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10810 }
10811 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10812 {
10813 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10814 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10815 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10816 || pCtx->fs.Attr.n.u4Type > 11
10817 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10818 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10819 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10820 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10821 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10822 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10823 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10824 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10825 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10826 }
10827 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10828 {
10829 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10830 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10831 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10832 || pCtx->gs.Attr.n.u4Type > 11
10833 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10834 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10835 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10836 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10837 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10838 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10839 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10840 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10841 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10842 }
10843 /* 64-bit capable CPUs. */
10844#if HC_ARCH_BITS == 64
10845 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10846 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10847 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10848 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10849 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10850 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10851 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10852 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10853 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10854 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10855 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10856#endif
10857 }
10858 else
10859 {
10860 /* V86 mode checks. */
10861 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10862 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10863 {
10864 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10865 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10866 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10867 }
10868 else
10869 {
10870 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10871 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10872 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10873 }
10874
10875 /* CS */
10876 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10877 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10878 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10879 /* SS */
10880 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10881 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10882 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10883 /* DS */
10884 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10885 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10886 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10887 /* ES */
10888 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10889 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10890 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10891 /* FS */
10892 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10893 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10894 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10895 /* GS */
10896 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10897 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10898 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10899 /* 64-bit capable CPUs. */
10900#if HC_ARCH_BITS == 64
10901 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10902 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10903 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10904 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10905 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10906 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
10907 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10908 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
10909 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10910 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
10911 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10912#endif
10913 }
10914
10915 /*
10916 * TR.
10917 */
10918 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10919 /* 64-bit capable CPUs. */
10920#if HC_ARCH_BITS == 64
10921 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10922#endif
10923 if (fLongModeGuest)
10924 {
10925 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10926 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10927 }
10928 else
10929 {
10930 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10931 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10932 VMX_IGS_TR_ATTR_TYPE_INVALID);
10933 }
10934 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10935 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10936 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10937 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10938 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10939 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10940 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10941 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10942
10943 /*
10944 * GDTR and IDTR.
10945 */
10946#if HC_ARCH_BITS == 64
10947 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10948 AssertRCBreak(rc);
10949 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10950
10951 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10952 AssertRCBreak(rc);
10953 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10954#endif
10955
10956 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10957 AssertRCBreak(rc);
10958 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10959
10960 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10961 AssertRCBreak(rc);
10962 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10963
10964 /*
10965 * Guest Non-Register State.
10966 */
10967 /* Activity State. */
10968 uint32_t u32ActivityState;
10969 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10970 AssertRCBreak(rc);
10971 HMVMX_CHECK_BREAK( !u32ActivityState
10972 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10973 VMX_IGS_ACTIVITY_STATE_INVALID);
10974 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10975 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10976 uint32_t u32IntrState;
10977 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10978 AssertRCBreak(rc);
10979 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10980 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10981 {
10982 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10983 }
10984
10985 /** @todo Activity state and injecting interrupts. Left as a todo since we
10986 * currently don't use activity states but ACTIVE. */
10987
10988 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10989 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10990
10991 /* Guest interruptibility-state. */
10992 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10993 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10994 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10995 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10996 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10997 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10998 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10999 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11000 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11001 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11002 {
11003 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11004 {
11005 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11006 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11007 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11008 }
11009 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11010 {
11011 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11012 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11013 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11014 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11015 }
11016 }
11017 /** @todo Assumes the processor is not in SMM. */
11018 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11019 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11020 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11021 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11022 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11023 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11024 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11025 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11026 {
11027 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11028 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11029 }
11030
11031 /* Pending debug exceptions. */
11032#if HC_ARCH_BITS == 64
11033 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11034 AssertRCBreak(rc);
11035 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11036 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11037 u32Val = u64Val; /* For pending debug exceptions checks below. */
11038#else
11039 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11040 AssertRCBreak(rc);
11041 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11042 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11043#endif
11044
11045 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11046 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11047 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11048 {
11049 if ( (u32Eflags & X86_EFL_TF)
11050 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11051 {
11052 /* Bit 14 is PendingDebug.BS. */
11053 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11054 }
11055 if ( !(u32Eflags & X86_EFL_TF)
11056 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11057 {
11058 /* Bit 14 is PendingDebug.BS. */
11059 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11060 }
11061 }
11062
11063 /* VMCS link pointer. */
11064 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11065 AssertRCBreak(rc);
11066 if (u64Val != UINT64_C(0xffffffffffffffff))
11067 {
11068 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11069 /** @todo Bits beyond the processor's physical-address width MBZ. */
11070 /** @todo 32-bit located in memory referenced by value of this field (as a
11071 * physical address) must contain the processor's VMCS revision ID. */
11072 /** @todo SMM checks. */
11073 }
11074
11075 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11076 * not using Nested Paging? */
11077 if ( pVM->hm.s.fNestedPaging
11078 && !fLongModeGuest
11079 && CPUMIsGuestInPAEModeEx(pCtx))
11080 {
11081 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11082 AssertRCBreak(rc);
11083 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11084
11085 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11086 AssertRCBreak(rc);
11087 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11088
11089 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11090 AssertRCBreak(rc);
11091 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11092
11093 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11094 AssertRCBreak(rc);
11095 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11096 }
11097
11098 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11099 if (uError == VMX_IGS_ERROR)
11100 uError = VMX_IGS_REASON_NOT_FOUND;
11101 } while (0);
11102
11103 pVCpu->hm.s.u32HMError = uError;
11104 return uError;
11105
11106#undef HMVMX_ERROR_BREAK
11107#undef HMVMX_CHECK_BREAK
11108}
11109
11110/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11111/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11112/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11113
11114/** @name VM-exit handlers.
11115 * @{
11116 */
11117
11118/**
11119 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11120 */
11121HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11122{
11123 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11125 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11126 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11127 return VINF_SUCCESS;
11128 return VINF_EM_RAW_INTERRUPT;
11129}
11130
11131
11132/**
11133 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11134 */
11135HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11136{
11137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11138 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11139
11140 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11141 AssertRCReturn(rc, rc);
11142
11143 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11144 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11145 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11146 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11147
11148 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11149 {
11150 /*
11151 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11152 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11153 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11154 *
11155 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11156 */
11157 VMXDispatchHostNmi();
11158 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11159 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11160 return VINF_SUCCESS;
11161 }
11162
11163 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11164 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11165 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11166 { /* likely */ }
11167 else
11168 {
11169 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11170 rcStrictRc1 = VINF_SUCCESS;
11171 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11172 return rcStrictRc1;
11173 }
11174
11175 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11176 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11177 switch (uIntType)
11178 {
11179 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11180 Assert(uVector == X86_XCPT_DB);
11181 /* no break */
11182 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11183 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11184 /* no break */
11185 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11186 {
11187 /*
11188 * If there's any exception caused as a result of event injection, go back to
11189 * the interpreter. The page-fault case is complicated and we manually handle
11190 * any currently pending event in hmR0VmxExitXcptPF. Nested #ACs are already
11191 * handled in hmR0VmxCheckExitDueToEventDelivery.
11192 */
11193 if (!pVCpu->hm.s.Event.fPending)
11194 { /* likely */ }
11195 else if ( uVector != X86_XCPT_PF
11196 && uVector != X86_XCPT_AC)
11197 {
11198 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
11199 rc = VERR_EM_INTERPRETER;
11200 break;
11201 }
11202
11203 switch (uVector)
11204 {
11205 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11206 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11207 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
11208 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11209 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11210 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11211 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11212
11213 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11214 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11215 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11216 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11217 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11218 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11219 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11220 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11221 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11222 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11223 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11224 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11225 default:
11226 {
11227 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11228 AssertRCReturn(rc, rc);
11229
11230 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11231 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11232 {
11233 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11234 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11235 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11236
11237 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11238 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11239 AssertRCReturn(rc, rc);
11240 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11241 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11242 0 /* GCPtrFaultAddress */);
11243 AssertRCReturn(rc, rc);
11244 }
11245 else
11246 {
11247 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11248 pVCpu->hm.s.u32HMError = uVector;
11249 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11250 }
11251 break;
11252 }
11253 }
11254 break;
11255 }
11256
11257 default:
11258 {
11259 pVCpu->hm.s.u32HMError = uExitIntInfo;
11260 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11261 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11262 break;
11263 }
11264 }
11265 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11266 return rc;
11267}
11268
11269
11270/**
11271 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11272 */
11273HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11274{
11275 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11276
11277 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11278 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11279
11280 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11282 return VINF_SUCCESS;
11283}
11284
11285
11286/**
11287 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11288 */
11289HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11290{
11291 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11292 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11293 {
11294 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11295 HMVMX_RETURN_UNEXPECTED_EXIT();
11296 }
11297
11298 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11299
11300 /*
11301 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11302 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11303 */
11304 uint32_t uIntrState = 0;
11305 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11306 AssertRCReturn(rc, rc);
11307
11308 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11309 if ( fBlockSti
11310 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11311 {
11312 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11313 }
11314
11315 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11316 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11317
11318 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11319 return VINF_SUCCESS;
11320}
11321
11322
11323/**
11324 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11325 */
11326HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11327{
11328 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11330 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11331}
11332
11333
11334/**
11335 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11336 */
11337HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11338{
11339 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11340 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11341 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11342}
11343
11344
11345/**
11346 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11347 */
11348HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11349{
11350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11351 PVM pVM = pVCpu->CTX_SUFF(pVM);
11352 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11353 if (RT_LIKELY(rc == VINF_SUCCESS))
11354 {
11355 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11356 Assert(pVmxTransient->cbInstr == 2);
11357 }
11358 else
11359 {
11360 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11361 rc = VERR_EM_INTERPRETER;
11362 }
11363 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11364 return rc;
11365}
11366
11367
11368/**
11369 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11370 */
11371HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11372{
11373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11374 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11375 AssertRCReturn(rc, rc);
11376
11377 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11378 return VINF_EM_RAW_EMULATE_INSTR;
11379
11380 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11381 HMVMX_RETURN_UNEXPECTED_EXIT();
11382}
11383
11384
11385/**
11386 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11387 */
11388HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11389{
11390 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11391 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11392 AssertRCReturn(rc, rc);
11393
11394 PVM pVM = pVCpu->CTX_SUFF(pVM);
11395 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11396 if (RT_LIKELY(rc == VINF_SUCCESS))
11397 {
11398 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11399 Assert(pVmxTransient->cbInstr == 2);
11400 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11401 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11402 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11403 }
11404 else
11405 rc = VERR_EM_INTERPRETER;
11406 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11407 return rc;
11408}
11409
11410
11411/**
11412 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11413 */
11414HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11415{
11416 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11417 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11418 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11419 AssertRCReturn(rc, rc);
11420
11421 PVM pVM = pVCpu->CTX_SUFF(pVM);
11422 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11423 if (RT_SUCCESS(rc))
11424 {
11425 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11426 Assert(pVmxTransient->cbInstr == 3);
11427 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11428 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11429 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11430 }
11431 else
11432 {
11433 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11434 rc = VERR_EM_INTERPRETER;
11435 }
11436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11437 return rc;
11438}
11439
11440
11441/**
11442 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11443 */
11444HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11445{
11446 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11447 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11448 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11449 AssertRCReturn(rc, rc);
11450
11451 PVM pVM = pVCpu->CTX_SUFF(pVM);
11452 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11453 if (RT_LIKELY(rc == VINF_SUCCESS))
11454 {
11455 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11456 Assert(pVmxTransient->cbInstr == 2);
11457 }
11458 else
11459 {
11460 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11461 rc = VERR_EM_INTERPRETER;
11462 }
11463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11464 return rc;
11465}
11466
11467
11468/**
11469 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11470 */
11471HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11472{
11473 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11474 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11475
11476 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11477 if (pVCpu->hm.s.fHypercallsEnabled)
11478 {
11479#if 0
11480 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11481#else
11482 /* Aggressive state sync. for now. */
11483 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11484 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11485 AssertRCReturn(rc, rc);
11486#endif
11487
11488 /* Perform the hypercall. */
11489 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11490 if (rcStrict == VINF_SUCCESS)
11491 {
11492 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11493 AssertRCReturn(rc, rc);
11494 }
11495 else
11496 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11497 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11498 || RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)));
11499
11500 /* If the hypercall changes anything other than guest's general-purpose registers,
11501 we would need to reload the guest changed bits here before VM-entry. */
11502 }
11503 else
11504 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11505
11506 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11507 if (RT_FAILURE(VBOXSTRICTRC_VAL(rcStrict)))
11508 {
11509 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11510 rcStrict = VINF_SUCCESS;
11511 }
11512
11513 return rcStrict;
11514}
11515
11516
11517/**
11518 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11519 */
11520HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11521{
11522 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11523 PVM pVM = pVCpu->CTX_SUFF(pVM);
11524 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11525
11526 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11527 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11528 AssertRCReturn(rc, rc);
11529
11530 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11531 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11532 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11533 else
11534 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11535 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11537 return rcStrict;
11538}
11539
11540
11541/**
11542 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11543 */
11544HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11545{
11546 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11547 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11548 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11549 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11550 AssertRCReturn(rc, rc);
11551
11552 PVM pVM = pVCpu->CTX_SUFF(pVM);
11553 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11554 if (RT_LIKELY(rc == VINF_SUCCESS))
11555 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11556 else
11557 {
11558 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11559 rc = VERR_EM_INTERPRETER;
11560 }
11561 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11562 return rc;
11563}
11564
11565
11566/**
11567 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11568 */
11569HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11570{
11571 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11572 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11573 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11574 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11575 AssertRCReturn(rc, rc);
11576
11577 PVM pVM = pVCpu->CTX_SUFF(pVM);
11578 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11579 rc = VBOXSTRICTRC_VAL(rc2);
11580 if (RT_LIKELY( rc == VINF_SUCCESS
11581 || rc == VINF_EM_HALT))
11582 {
11583 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11584 AssertRCReturn(rc3, rc3);
11585
11586 if ( rc == VINF_EM_HALT
11587 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11588 {
11589 rc = VINF_SUCCESS;
11590 }
11591 }
11592 else
11593 {
11594 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11595 rc = VERR_EM_INTERPRETER;
11596 }
11597 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11598 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11599 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11600 return rc;
11601}
11602
11603
11604/**
11605 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11606 */
11607HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11608{
11609 /*
11610 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
11611 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
11612 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
11613 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
11614 */
11615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11616 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11617 HMVMX_RETURN_UNEXPECTED_EXIT();
11618}
11619
11620
11621/**
11622 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11623 */
11624HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11625{
11626 /*
11627 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
11628 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
11629 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
11630 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
11631 */
11632 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11633 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11634 HMVMX_RETURN_UNEXPECTED_EXIT();
11635}
11636
11637
11638/**
11639 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11640 */
11641HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11642{
11643 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11644 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11645 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11646 HMVMX_RETURN_UNEXPECTED_EXIT();
11647}
11648
11649
11650/**
11651 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11652 */
11653HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11654{
11655 /*
11656 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
11657 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
11658 * See Intel spec. 25.3 "Other Causes of VM-exits".
11659 */
11660 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11661 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11662 HMVMX_RETURN_UNEXPECTED_EXIT();
11663}
11664
11665
11666/**
11667 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11668 * VM-exit.
11669 */
11670HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11671{
11672 /*
11673 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11674 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11675 *
11676 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11677 * See Intel spec. "23.8 Restrictions on VMX operation".
11678 */
11679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11680 return VINF_SUCCESS;
11681}
11682
11683
11684/**
11685 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11686 * VM-exit.
11687 */
11688HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11689{
11690 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11691 return VINF_EM_RESET;
11692}
11693
11694
11695/**
11696 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11697 */
11698HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11699{
11700 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11701 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11702
11703 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11704 AssertRCReturn(rc, rc);
11705
11706 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11707 rc = VINF_SUCCESS;
11708 else
11709 rc = VINF_EM_HALT;
11710
11711 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11712 if (rc != VINF_SUCCESS)
11713 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11714 return rc;
11715}
11716
11717
11718/**
11719 * VM-exit handler for instructions that result in a \#UD exception delivered to
11720 * the guest.
11721 */
11722HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11723{
11724 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11725 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11726 return VINF_SUCCESS;
11727}
11728
11729
11730/**
11731 * VM-exit handler for expiry of the VMX preemption timer.
11732 */
11733HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11734{
11735 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11736
11737 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11738 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11739
11740 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11741 PVM pVM = pVCpu->CTX_SUFF(pVM);
11742 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11743 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11744 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11745}
11746
11747
11748/**
11749 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11750 */
11751HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11752{
11753 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11754
11755 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11756 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11757 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11758 AssertRCReturn(rc, rc);
11759
11760 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11761 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
11762
11763 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11764
11765 return rcStrict;
11766}
11767
11768
11769/**
11770 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11771 */
11772HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11773{
11774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11775
11776 /* The guest should not invalidate the host CPU's TLBs, fallback to interpreter. */
11777 /** @todo implement EMInterpretInvpcid() */
11778 return VERR_EM_INTERPRETER;
11779}
11780
11781
11782/**
11783 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11784 * Error VM-exit.
11785 */
11786HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11787{
11788 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11789 AssertRCReturn(rc, rc);
11790
11791 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11792 AssertRCReturn(rc, rc);
11793
11794 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11795 NOREF(uInvalidReason);
11796
11797#ifdef VBOX_STRICT
11798 uint32_t uIntrState;
11799 RTHCUINTREG uHCReg;
11800 uint64_t u64Val;
11801 uint32_t u32Val;
11802
11803 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11804 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11805 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11806 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11807 AssertRCReturn(rc, rc);
11808
11809 Log4(("uInvalidReason %u\n", uInvalidReason));
11810 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11811 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11812 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11813 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
11814
11815 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11816 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11817 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11818 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11819 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11820 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11821 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11822 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11823 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11824 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11825 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11826 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11827#else
11828 NOREF(pVmxTransient);
11829#endif
11830
11831 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11832 return VERR_VMX_INVALID_GUEST_STATE;
11833}
11834
11835
11836/**
11837 * VM-exit handler for VM-entry failure due to an MSR-load
11838 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11839 */
11840HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11841{
11842 NOREF(pVmxTransient);
11843 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11844 HMVMX_RETURN_UNEXPECTED_EXIT();
11845}
11846
11847
11848/**
11849 * VM-exit handler for VM-entry failure due to a machine-check event
11850 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11851 */
11852HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11853{
11854 NOREF(pVmxTransient);
11855 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11856 HMVMX_RETURN_UNEXPECTED_EXIT();
11857}
11858
11859
11860/**
11861 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11862 * theory.
11863 */
11864HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11865{
11866 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11867 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11868 return VERR_VMX_UNDEFINED_EXIT_CODE;
11869}
11870
11871
11872/**
11873 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11874 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11875 * Conditional VM-exit.
11876 */
11877HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11878{
11879 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11880
11881 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11883 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11884 return VERR_EM_INTERPRETER;
11885 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11886 HMVMX_RETURN_UNEXPECTED_EXIT();
11887}
11888
11889
11890/**
11891 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11892 */
11893HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11894{
11895 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11896
11897 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11898 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
11899 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11900 return VERR_EM_INTERPRETER;
11901 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11902 HMVMX_RETURN_UNEXPECTED_EXIT();
11903}
11904
11905
11906/**
11907 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11908 */
11909HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11910{
11911 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11912
11913 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11914 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11915 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11916 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11917 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11918 {
11919 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11920 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11921 }
11922 AssertRCReturn(rc, rc);
11923 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
11924
11925#ifdef VBOX_STRICT
11926 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11927 {
11928 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11929 && pMixedCtx->ecx != MSR_K6_EFER)
11930 {
11931 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11932 pMixedCtx->ecx));
11933 HMVMX_RETURN_UNEXPECTED_EXIT();
11934 }
11935 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11936 {
11937 VMXMSREXITREAD enmRead;
11938 VMXMSREXITWRITE enmWrite;
11939 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11940 AssertRCReturn(rc2, rc2);
11941 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11942 {
11943 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11944 HMVMX_RETURN_UNEXPECTED_EXIT();
11945 }
11946 }
11947 }
11948#endif
11949
11950 PVM pVM = pVCpu->CTX_SUFF(pVM);
11951 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11952 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11953 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11955 if (RT_SUCCESS(rc))
11956 {
11957 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11958 Assert(pVmxTransient->cbInstr == 2);
11959 }
11960 return rc;
11961}
11962
11963
11964/**
11965 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11966 */
11967HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11968{
11969 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11970 PVM pVM = pVCpu->CTX_SUFF(pVM);
11971 int rc = VINF_SUCCESS;
11972
11973 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11974 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11975 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11976 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11977 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11978 {
11979 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
11980 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
11981 }
11982 AssertRCReturn(rc, rc);
11983 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11984
11985 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11986 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11987 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11988
11989 if (RT_SUCCESS(rc))
11990 {
11991 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11992
11993 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11994 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11995 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
11996 {
11997 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11998 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11999 EMInterpretWrmsr() changes it. */
12000 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12001 }
12002 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12003 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12004 else if (pMixedCtx->ecx == MSR_K6_EFER)
12005 {
12006 /*
12007 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12008 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12009 * the other bits as well, SCE and NXE. See @bugref{7368}.
12010 */
12011 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12012 }
12013
12014 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12015 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12016 {
12017 switch (pMixedCtx->ecx)
12018 {
12019 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
12020 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
12021 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
12022 case MSR_K8_FS_BASE: /* no break */
12023 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12024 case MSR_K6_EFER: /* already handled above */ break;
12025 default:
12026 {
12027 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12028 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12029 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12030 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
12031 break;
12032 }
12033 }
12034 }
12035#ifdef VBOX_STRICT
12036 else
12037 {
12038 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12039 switch (pMixedCtx->ecx)
12040 {
12041 case MSR_IA32_SYSENTER_CS:
12042 case MSR_IA32_SYSENTER_EIP:
12043 case MSR_IA32_SYSENTER_ESP:
12044 case MSR_K8_FS_BASE:
12045 case MSR_K8_GS_BASE:
12046 {
12047 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12048 HMVMX_RETURN_UNEXPECTED_EXIT();
12049 }
12050
12051 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12052 default:
12053 {
12054 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12055 {
12056 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12057 if (pMixedCtx->ecx != MSR_K6_EFER)
12058 {
12059 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12060 pMixedCtx->ecx));
12061 HMVMX_RETURN_UNEXPECTED_EXIT();
12062 }
12063 }
12064
12065 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12066 {
12067 VMXMSREXITREAD enmRead;
12068 VMXMSREXITWRITE enmWrite;
12069 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12070 AssertRCReturn(rc2, rc2);
12071 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12072 {
12073 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12074 HMVMX_RETURN_UNEXPECTED_EXIT();
12075 }
12076 }
12077 break;
12078 }
12079 }
12080 }
12081#endif /* VBOX_STRICT */
12082 }
12083 return rc;
12084}
12085
12086
12087/**
12088 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12089 */
12090HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12091{
12092 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12093
12094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12095 return VINF_EM_RAW_INTERRUPT;
12096}
12097
12098
12099/**
12100 * VM-exit handler for when the TPR value is lowered below the specified
12101 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12102 */
12103HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12104{
12105 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12106 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12107
12108 /*
12109 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
12110 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
12111 * resume guest execution.
12112 */
12113 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12115 return VINF_SUCCESS;
12116}
12117
12118
12119/**
12120 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12121 * VM-exit.
12122 *
12123 * @retval VINF_SUCCESS when guest execution can continue.
12124 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12125 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12126 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12127 * interpreter.
12128 */
12129HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12130{
12131 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12132 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12133 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12134 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12135 AssertRCReturn(rc, rc);
12136
12137 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12138 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12139 PVM pVM = pVCpu->CTX_SUFF(pVM);
12140 VBOXSTRICTRC rcStrict;
12141 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12142 switch (uAccessType)
12143 {
12144 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12145 {
12146 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12147 AssertRCReturn(rc, rc);
12148
12149 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12150 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12151 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12152 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12153 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12154 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12155 {
12156 case 0: /* CR0 */
12157 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12158 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12159 break;
12160 case 2: /* CR2 */
12161 /* Nothing to do here, CR2 it's not part of the VMCS. */
12162 break;
12163 case 3: /* CR3 */
12164 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12165 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12166 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12167 break;
12168 case 4: /* CR4 */
12169 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12170 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12171 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12172 break;
12173 case 8: /* CR8 */
12174 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12175 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12176 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
12177 break;
12178 default:
12179 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12180 break;
12181 }
12182
12183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12184 break;
12185 }
12186
12187 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12188 {
12189 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12190 AssertRCReturn(rc, rc);
12191
12192 Assert( !pVM->hm.s.fNestedPaging
12193 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12194 || pVCpu->hm.s.fUsingDebugLoop
12195 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12196
12197 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12198 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12199 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12200
12201 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12202 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12203 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12204 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12205 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12206 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12207 VBOXSTRICTRC_VAL(rcStrict)));
12208 break;
12209 }
12210
12211 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12212 {
12213 AssertRCReturn(rc, rc);
12214 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12215 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12216 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12217 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12218 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12219 break;
12220 }
12221
12222 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12223 {
12224 AssertRCReturn(rc, rc);
12225 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12226 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12227 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12228 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12229 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12230 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12231 break;
12232 }
12233
12234 default:
12235 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12236 VERR_VMX_UNEXPECTED_EXCEPTION);
12237 }
12238
12239 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12240 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12241 NOREF(pVM);
12242 return rcStrict;
12243}
12244
12245
12246/**
12247 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12248 * VM-exit.
12249 */
12250HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12251{
12252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12253 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12254
12255 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12256 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12257 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12258 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12259 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12260 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12261 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12262 AssertRCReturn(rc2, rc2);
12263
12264 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12265 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12266 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12267 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12268 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12269 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12270 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12271 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12272 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12273
12274 /* I/O operation lookup arrays. */
12275 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12276 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12277
12278 VBOXSTRICTRC rcStrict;
12279 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12280 uint32_t const cbInstr = pVmxTransient->cbInstr;
12281 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12282 PVM pVM = pVCpu->CTX_SUFF(pVM);
12283 if (fIOString)
12284 {
12285#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12286 See @bugref{5752#c158}. Should work now. */
12287 /*
12288 * INS/OUTS - I/O String instruction.
12289 *
12290 * Use instruction-information if available, otherwise fall back on
12291 * interpreting the instruction.
12292 */
12293 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12294 fIOWrite ? 'w' : 'r'));
12295 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12296 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12297 {
12298 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12299 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12300 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12301 AssertRCReturn(rc2, rc2);
12302 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12303 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12304 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12305 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12306 if (fIOWrite)
12307 {
12308 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12309 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12310 }
12311 else
12312 {
12313 /*
12314 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12315 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12316 * See Intel Instruction spec. for "INS".
12317 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12318 */
12319 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12320 }
12321 }
12322 else
12323 {
12324 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12325 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12326 AssertRCReturn(rc2, rc2);
12327 rcStrict = IEMExecOne(pVCpu);
12328 }
12329 /** @todo IEM needs to be setting these flags somehow. */
12330 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12331 fUpdateRipAlready = true;
12332#else
12333 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12334 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12335 if (RT_SUCCESS(rcStrict))
12336 {
12337 if (fIOWrite)
12338 {
12339 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12340 (DISCPUMODE)pDis->uAddrMode, cbValue);
12341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12342 }
12343 else
12344 {
12345 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12346 (DISCPUMODE)pDis->uAddrMode, cbValue);
12347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12348 }
12349 }
12350 else
12351 {
12352 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12353 pMixedCtx->rip));
12354 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12355 }
12356#endif
12357 }
12358 else
12359 {
12360 /*
12361 * IN/OUT - I/O instruction.
12362 */
12363 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12364 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12365 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12366 if (fIOWrite)
12367 {
12368 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12369 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12370 }
12371 else
12372 {
12373 uint32_t u32Result = 0;
12374 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12375 if (IOM_SUCCESS(rcStrict))
12376 {
12377 /* Save result of I/O IN instr. in AL/AX/EAX. */
12378 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12379 }
12380 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12381 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12382 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12383 }
12384 }
12385
12386 if (IOM_SUCCESS(rcStrict))
12387 {
12388 if (!fUpdateRipAlready)
12389 {
12390 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12391 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12392 }
12393
12394 /*
12395 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12396 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12397 */
12398 if (fIOString)
12399 {
12400 /** @todo Single-step for INS/OUTS with REP prefix? */
12401 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12402 }
12403 else if ( !fDbgStepping
12404 && fGstStepping)
12405 {
12406 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12407 }
12408
12409 /*
12410 * If any I/O breakpoints are armed, we need to check if one triggered
12411 * and take appropriate action.
12412 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12413 */
12414 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12415 AssertRCReturn(rc2, rc2);
12416
12417 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12418 * execution engines about whether hyper BPs and such are pending. */
12419 uint32_t const uDr7 = pMixedCtx->dr[7];
12420 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12421 && X86_DR7_ANY_RW_IO(uDr7)
12422 && (pMixedCtx->cr4 & X86_CR4_DE))
12423 || DBGFBpIsHwIoArmed(pVM)))
12424 {
12425 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12426
12427 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12428 VMMRZCallRing3Disable(pVCpu);
12429 HM_DISABLE_PREEMPT();
12430
12431 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12432
12433 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12434 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12435 {
12436 /* Raise #DB. */
12437 if (fIsGuestDbgActive)
12438 ASMSetDR6(pMixedCtx->dr[6]);
12439 if (pMixedCtx->dr[7] != uDr7)
12440 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12441
12442 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12443 }
12444 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12445 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12446 else if ( rcStrict2 != VINF_SUCCESS
12447 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12448 rcStrict = rcStrict2;
12449 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12450
12451 HM_RESTORE_PREEMPT();
12452 VMMRZCallRing3Enable(pVCpu);
12453 }
12454 }
12455
12456#ifdef VBOX_STRICT
12457 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12458 Assert(!fIOWrite);
12459 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12460 Assert(fIOWrite);
12461 else
12462 {
12463#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12464 * statuses, that the VMM device and some others may return. See
12465 * IOM_SUCCESS() for guidance. */
12466 AssertMsg( RT_FAILURE(rcStrict)
12467 || rcStrict == VINF_SUCCESS
12468 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12469 || rcStrict == VINF_EM_DBG_BREAKPOINT
12470 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12471 || rcStrict == VINF_EM_RAW_TO_R3
12472 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12473#endif
12474 }
12475#endif
12476
12477 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12478 return rcStrict;
12479}
12480
12481
12482/**
12483 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12484 * VM-exit.
12485 */
12486HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12487{
12488 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12489
12490 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12491 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12492 AssertRCReturn(rc, rc);
12493 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12494 {
12495 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12496 AssertRCReturn(rc, rc);
12497 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12498 {
12499 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12500
12501 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12502 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12503
12504 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
12505 Assert(!pVCpu->hm.s.Event.fPending);
12506 pVCpu->hm.s.Event.fPending = true;
12507 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
12508 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12509 AssertRCReturn(rc, rc);
12510 if (fErrorCodeValid)
12511 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12512 else
12513 pVCpu->hm.s.Event.u32ErrCode = 0;
12514 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12515 && uVector == X86_XCPT_PF)
12516 {
12517 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
12518 }
12519
12520 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12521 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12522 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12523 }
12524 }
12525
12526 /* Fall back to the interpreter to emulate the task-switch. */
12527 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12528 return VERR_EM_INTERPRETER;
12529}
12530
12531
12532/**
12533 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12534 */
12535HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12536{
12537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12538 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12539 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12540 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12541 AssertRCReturn(rc, rc);
12542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12543 return VINF_EM_DBG_STEPPED;
12544}
12545
12546
12547/**
12548 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12549 */
12550HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12551{
12552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12553
12554 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12555
12556 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12557 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12558 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12559 {
12560 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12561 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12562 {
12563 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12564 return VERR_EM_INTERPRETER;
12565 }
12566 }
12567 else
12568 {
12569 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12570 rcStrict1 = VINF_SUCCESS;
12571 return rcStrict1;
12572 }
12573
12574#if 0
12575 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
12576 * just sync the whole thing. */
12577 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12578#else
12579 /* Aggressive state sync. for now. */
12580 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12581 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12582 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12583#endif
12584 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12585 AssertRCReturn(rc, rc);
12586
12587 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12588 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12589 VBOXSTRICTRC rcStrict2;
12590 switch (uAccessType)
12591 {
12592 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12593 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12594 {
12595 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12596 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != 0x80,
12597 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12598
12599 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
12600 GCPhys &= PAGE_BASE_GC_MASK;
12601 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12602 PVM pVM = pVCpu->CTX_SUFF(pVM);
12603 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12604 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12605
12606 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12607 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12608 CPUMCTX2CORE(pMixedCtx), GCPhys);
12609 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
12610 if ( rcStrict2 == VINF_SUCCESS
12611 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12612 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12613 {
12614 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12615 | HM_CHANGED_GUEST_RSP
12616 | HM_CHANGED_GUEST_RFLAGS
12617 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12618 rcStrict2 = VINF_SUCCESS;
12619 }
12620 break;
12621 }
12622
12623 default:
12624 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
12625 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12626 break;
12627 }
12628
12629 if (rcStrict2 != VINF_SUCCESS)
12630 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12631 return rcStrict2;
12632}
12633
12634
12635/**
12636 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12637 * VM-exit.
12638 */
12639HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12640{
12641 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12642
12643 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12644 if (pVmxTransient->fWasGuestDebugStateActive)
12645 {
12646 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12647 HMVMX_RETURN_UNEXPECTED_EXIT();
12648 }
12649
12650 if ( !pVCpu->hm.s.fSingleInstruction
12651 && !pVmxTransient->fWasHyperDebugStateActive)
12652 {
12653 Assert(!DBGFIsStepping(pVCpu));
12654 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12655
12656 /* Don't intercept MOV DRx any more. */
12657 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12658 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12659 AssertRCReturn(rc, rc);
12660
12661 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12662 VMMRZCallRing3Disable(pVCpu);
12663 HM_DISABLE_PREEMPT();
12664
12665 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12666 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12667 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12668
12669 HM_RESTORE_PREEMPT();
12670 VMMRZCallRing3Enable(pVCpu);
12671
12672#ifdef VBOX_WITH_STATISTICS
12673 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12674 AssertRCReturn(rc, rc);
12675 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12677 else
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12679#endif
12680 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12681 return VINF_SUCCESS;
12682 }
12683
12684 /*
12685 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12686 * Update the segment registers and DR7 from the CPU.
12687 */
12688 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12689 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12690 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12691 AssertRCReturn(rc, rc);
12692 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12693
12694 PVM pVM = pVCpu->CTX_SUFF(pVM);
12695 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
12696 {
12697 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12698 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
12699 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
12700 if (RT_SUCCESS(rc))
12701 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12703 }
12704 else
12705 {
12706 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12707 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
12708 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
12709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12710 }
12711
12712 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12713 if (RT_SUCCESS(rc))
12714 {
12715 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12716 AssertRCReturn(rc2, rc2);
12717 return VINF_SUCCESS;
12718 }
12719 return rc;
12720}
12721
12722
12723/**
12724 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12725 * Conditional VM-exit.
12726 */
12727HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12728{
12729 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12730 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12731
12732 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12733 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12734 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12735 {
12736 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12737 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12738 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12739 {
12740 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12741 return VERR_EM_INTERPRETER;
12742 }
12743 }
12744 else
12745 {
12746 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12747 rcStrict1 = VINF_SUCCESS;
12748 return rcStrict1;
12749 }
12750
12751 RTGCPHYS GCPhys = 0;
12752 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12753
12754#if 0
12755 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12756#else
12757 /* Aggressive state sync. for now. */
12758 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12759 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12760 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12761#endif
12762 AssertRCReturn(rc, rc);
12763
12764 /*
12765 * If we succeed, resume guest execution.
12766 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12767 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12768 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12769 * weird case. See @bugref{6043}.
12770 */
12771 PVM pVM = pVCpu->CTX_SUFF(pVM);
12772 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12773 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
12774 if ( rcStrict2 == VINF_SUCCESS
12775 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12776 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12777 {
12778 /* Successfully handled MMIO operation. */
12779 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12780 | HM_CHANGED_GUEST_RSP
12781 | HM_CHANGED_GUEST_RFLAGS
12782 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12783 return VINF_SUCCESS;
12784 }
12785 return rcStrict2;
12786}
12787
12788
12789/**
12790 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12791 * VM-exit.
12792 */
12793HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12794{
12795 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12796 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12797
12798 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12799 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12800 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12801 {
12802 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12803 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12804 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12805 }
12806 else
12807 {
12808 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12809 rcStrict1 = VINF_SUCCESS;
12810 return rcStrict1;
12811 }
12812
12813 RTGCPHYS GCPhys = 0;
12814 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12815 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12816#if 0
12817 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
12818#else
12819 /* Aggressive state sync. for now. */
12820 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
12821 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12822 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12823#endif
12824 AssertRCReturn(rc, rc);
12825
12826 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12827 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12828
12829 RTGCUINT uErrorCode = 0;
12830 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
12831 uErrorCode |= X86_TRAP_PF_ID;
12832 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
12833 uErrorCode |= X86_TRAP_PF_RW;
12834 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
12835 uErrorCode |= X86_TRAP_PF_P;
12836
12837 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12838
12839 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12840 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12841
12842 /* Handle the pagefault trap for the nested shadow table. */
12843 PVM pVM = pVCpu->CTX_SUFF(pVM);
12844 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12845 TRPMResetTrap(pVCpu);
12846
12847 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12848 if ( rcStrict2 == VINF_SUCCESS
12849 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12850 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12851 {
12852 /* Successfully synced our nested page tables. */
12853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12854 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12855 | HM_CHANGED_GUEST_RSP
12856 | HM_CHANGED_GUEST_RFLAGS);
12857 return VINF_SUCCESS;
12858 }
12859
12860 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12861 return rcStrict2;
12862}
12863
12864/** @} */
12865
12866/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12867/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12868/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12869
12870/** @name VM-exit exception handlers.
12871 * @{
12872 */
12873
12874/**
12875 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12876 */
12877static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12878{
12879 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12880 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12881
12882 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12883 AssertRCReturn(rc, rc);
12884
12885 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12886 {
12887 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12888 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12889
12890 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12891 * provides VM-exit instruction length. If this causes problem later,
12892 * disassemble the instruction like it's done on AMD-V. */
12893 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12894 AssertRCReturn(rc2, rc2);
12895 return rc;
12896 }
12897
12898 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12899 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12900 return rc;
12901}
12902
12903
12904/**
12905 * VM-exit exception handler for \#BP (Breakpoint exception).
12906 */
12907static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12908{
12909 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12911
12912 /** @todo Try optimize this by not saving the entire guest state unless
12913 * really needed. */
12914 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12915 AssertRCReturn(rc, rc);
12916
12917 PVM pVM = pVCpu->CTX_SUFF(pVM);
12918 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12919 if (rc == VINF_EM_RAW_GUEST_TRAP)
12920 {
12921 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12922 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12923 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12924 AssertRCReturn(rc, rc);
12925
12926 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12927 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12928 }
12929
12930 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12931 return rc;
12932}
12933
12934
12935/**
12936 * VM-exit exception handler for \#AC (alignment check exception).
12937 */
12938static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12939{
12940 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12941
12942 /*
12943 * Re-inject it. We'll detect any nesting before getting here.
12944 */
12945 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12946 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12947 AssertRCReturn(rc, rc);
12948 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
12949
12950 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12951 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12952 return VINF_SUCCESS;
12953}
12954
12955
12956/**
12957 * VM-exit exception handler for \#DB (Debug exception).
12958 */
12959static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12960{
12961 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12962 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12963 Log6(("XcptDB\n"));
12964
12965 /*
12966 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12967 * for processing.
12968 */
12969 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12970 AssertRCReturn(rc, rc);
12971
12972 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12973 uint64_t uDR6 = X86_DR6_INIT_VAL;
12974 uDR6 |= ( pVmxTransient->uExitQualification
12975 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12976
12977 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12978 if (rc == VINF_EM_RAW_GUEST_TRAP)
12979 {
12980 /*
12981 * The exception was for the guest. Update DR6, DR7.GD and
12982 * IA32_DEBUGCTL.LBR before forwarding it.
12983 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12984 */
12985 VMMRZCallRing3Disable(pVCpu);
12986 HM_DISABLE_PREEMPT();
12987
12988 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12989 pMixedCtx->dr[6] |= uDR6;
12990 if (CPUMIsGuestDebugStateActive(pVCpu))
12991 ASMSetDR6(pMixedCtx->dr[6]);
12992
12993 HM_RESTORE_PREEMPT();
12994 VMMRZCallRing3Enable(pVCpu);
12995
12996 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12997 AssertRCReturn(rc, rc);
12998
12999 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13000 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13001
13002 /* Paranoia. */
13003 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13004 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13005
13006 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13007 AssertRCReturn(rc, rc);
13008
13009 /*
13010 * Raise #DB in the guest.
13011 *
13012 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13013 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
13014 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
13015 *
13016 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
13017 */
13018 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13019 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13020 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13021 AssertRCReturn(rc, rc);
13022 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13023 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13024 return VINF_SUCCESS;
13025 }
13026
13027 /*
13028 * Not a guest trap, must be a hypervisor related debug event then.
13029 * Update DR6 in case someone is interested in it.
13030 */
13031 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13032 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13033 CPUMSetHyperDR6(pVCpu, uDR6);
13034
13035 return rc;
13036}
13037
13038
13039/**
13040 * VM-exit exception handler for \#NM (Device-not-available exception: floating
13041 * point exception).
13042 */
13043static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13044{
13045 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13046
13047 /* We require CR0 and EFER. EFER is always up-to-date. */
13048 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13049 AssertRCReturn(rc, rc);
13050
13051 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
13052 VMMRZCallRing3Disable(pVCpu);
13053 HM_DISABLE_PREEMPT();
13054
13055 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
13056 if (pVmxTransient->fWasGuestFPUStateActive)
13057 {
13058 rc = VINF_EM_RAW_GUEST_TRAP;
13059 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
13060 }
13061 else
13062 {
13063#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13064 Assert(!pVmxTransient->fWasGuestFPUStateActive || pVCpu->hm.s.fUsingDebugLoop);
13065#endif
13066 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu);
13067 Assert( rc == VINF_EM_RAW_GUEST_TRAP
13068 || ((rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED) && CPUMIsGuestFPUStateActive(pVCpu)));
13069 if (rc == VINF_CPUM_HOST_CR0_MODIFIED)
13070 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
13071 }
13072
13073 HM_RESTORE_PREEMPT();
13074 VMMRZCallRing3Enable(pVCpu);
13075
13076 if (rc == VINF_SUCCESS || rc == VINF_CPUM_HOST_CR0_MODIFIED)
13077 {
13078 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
13079 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
13080 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
13081 pVCpu->hm.s.fPreloadGuestFpu = true;
13082 }
13083 else
13084 {
13085 /* Forward #NM to the guest. */
13086 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
13087 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13088 AssertRCReturn(rc, rc);
13089 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13090 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
13091 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
13092 }
13093
13094 return VINF_SUCCESS;
13095}
13096
13097
13098/**
13099 * VM-exit exception handler for \#GP (General-protection exception).
13100 *
13101 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13102 */
13103static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13104{
13105 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13107
13108 int rc;
13109 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13110 { /* likely */ }
13111 else
13112 {
13113#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13114 Assert(pVCpu->hm.s.fUsingDebugLoop);
13115#endif
13116 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13117 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13118 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13119 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13120 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13121 AssertRCReturn(rc, rc);
13122 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13123 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13124 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13125 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13126 return rc;
13127 }
13128
13129 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13130 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13131
13132 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13133 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13134 AssertRCReturn(rc, rc);
13135
13136 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13137 uint32_t cbOp = 0;
13138 PVM pVM = pVCpu->CTX_SUFF(pVM);
13139 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13140 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13141 if (RT_SUCCESS(rc))
13142 {
13143 rc = VINF_SUCCESS;
13144 Assert(cbOp == pDis->cbInstr);
13145 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13146 switch (pDis->pCurInstr->uOpcode)
13147 {
13148 case OP_CLI:
13149 {
13150 pMixedCtx->eflags.Bits.u1IF = 0;
13151 pMixedCtx->eflags.Bits.u1RF = 0;
13152 pMixedCtx->rip += pDis->cbInstr;
13153 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13154 if ( !fDbgStepping
13155 && pMixedCtx->eflags.Bits.u1TF)
13156 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13158 break;
13159 }
13160
13161 case OP_STI:
13162 {
13163 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13164 pMixedCtx->eflags.Bits.u1IF = 1;
13165 pMixedCtx->eflags.Bits.u1RF = 0;
13166 pMixedCtx->rip += pDis->cbInstr;
13167 if (!fOldIF)
13168 {
13169 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13170 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13171 }
13172 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13173 if ( !fDbgStepping
13174 && pMixedCtx->eflags.Bits.u1TF)
13175 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13177 break;
13178 }
13179
13180 case OP_HLT:
13181 {
13182 rc = VINF_EM_HALT;
13183 pMixedCtx->rip += pDis->cbInstr;
13184 pMixedCtx->eflags.Bits.u1RF = 0;
13185 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13187 break;
13188 }
13189
13190 case OP_POPF:
13191 {
13192 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13193 uint32_t cbParm;
13194 uint32_t uMask;
13195 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13196 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13197 {
13198 cbParm = 4;
13199 uMask = 0xffffffff;
13200 }
13201 else
13202 {
13203 cbParm = 2;
13204 uMask = 0xffff;
13205 }
13206
13207 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13208 RTGCPTR GCPtrStack = 0;
13209 X86EFLAGS Eflags;
13210 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13211 &GCPtrStack);
13212 if (RT_SUCCESS(rc))
13213 {
13214 Assert(sizeof(Eflags.u32) >= cbParm);
13215 Eflags.u32 = 0;
13216 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13217 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13218 }
13219 if (RT_FAILURE(rc))
13220 {
13221 rc = VERR_EM_INTERPRETER;
13222 break;
13223 }
13224 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13225 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13226 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13227 pMixedCtx->esp += cbParm;
13228 pMixedCtx->esp &= uMask;
13229 pMixedCtx->rip += pDis->cbInstr;
13230 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13231 | HM_CHANGED_GUEST_RSP
13232 | HM_CHANGED_GUEST_RFLAGS);
13233 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13234 POPF restores EFLAGS.TF. */
13235 if ( !fDbgStepping
13236 && fGstStepping)
13237 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13238 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13239 break;
13240 }
13241
13242 case OP_PUSHF:
13243 {
13244 uint32_t cbParm;
13245 uint32_t uMask;
13246 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13247 {
13248 cbParm = 4;
13249 uMask = 0xffffffff;
13250 }
13251 else
13252 {
13253 cbParm = 2;
13254 uMask = 0xffff;
13255 }
13256
13257 /* Get the stack pointer & push the contents of eflags onto the stack. */
13258 RTGCPTR GCPtrStack = 0;
13259 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13260 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13261 if (RT_FAILURE(rc))
13262 {
13263 rc = VERR_EM_INTERPRETER;
13264 break;
13265 }
13266 X86EFLAGS Eflags = pMixedCtx->eflags;
13267 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13268 Eflags.Bits.u1RF = 0;
13269 Eflags.Bits.u1VM = 0;
13270
13271 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13272 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13273 {
13274 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13275 rc = VERR_EM_INTERPRETER;
13276 break;
13277 }
13278 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13279 pMixedCtx->esp -= cbParm;
13280 pMixedCtx->esp &= uMask;
13281 pMixedCtx->rip += pDis->cbInstr;
13282 pMixedCtx->eflags.Bits.u1RF = 0;
13283 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13284 | HM_CHANGED_GUEST_RSP
13285 | HM_CHANGED_GUEST_RFLAGS);
13286 if ( !fDbgStepping
13287 && pMixedCtx->eflags.Bits.u1TF)
13288 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13289 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13290 break;
13291 }
13292
13293 case OP_IRET:
13294 {
13295 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13296 * instruction reference. */
13297 RTGCPTR GCPtrStack = 0;
13298 uint32_t uMask = 0xffff;
13299 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13300 uint16_t aIretFrame[3];
13301 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13302 {
13303 rc = VERR_EM_INTERPRETER;
13304 break;
13305 }
13306 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13307 &GCPtrStack);
13308 if (RT_SUCCESS(rc))
13309 {
13310 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13311 PGMACCESSORIGIN_HM));
13312 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13313 }
13314 if (RT_FAILURE(rc))
13315 {
13316 rc = VERR_EM_INTERPRETER;
13317 break;
13318 }
13319 pMixedCtx->eip = 0;
13320 pMixedCtx->ip = aIretFrame[0];
13321 pMixedCtx->cs.Sel = aIretFrame[1];
13322 pMixedCtx->cs.ValidSel = aIretFrame[1];
13323 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13324 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13325 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13326 pMixedCtx->sp += sizeof(aIretFrame);
13327 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13328 | HM_CHANGED_GUEST_SEGMENT_REGS
13329 | HM_CHANGED_GUEST_RSP
13330 | HM_CHANGED_GUEST_RFLAGS);
13331 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13332 if ( !fDbgStepping
13333 && fGstStepping)
13334 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13335 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13337 break;
13338 }
13339
13340 case OP_INT:
13341 {
13342 uint16_t uVector = pDis->Param1.uValue & 0xff;
13343 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13344 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13345 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13346 break;
13347 }
13348
13349 case OP_INTO:
13350 {
13351 if (pMixedCtx->eflags.Bits.u1OF)
13352 {
13353 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13354 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13356 }
13357 else
13358 {
13359 pMixedCtx->eflags.Bits.u1RF = 0;
13360 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13361 }
13362 break;
13363 }
13364
13365 default:
13366 {
13367 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13368 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13369 EMCODETYPE_SUPERVISOR);
13370 rc = VBOXSTRICTRC_VAL(rc2);
13371 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13372 /** @todo We have to set pending-debug exceptions here when the guest is
13373 * single-stepping depending on the instruction that was interpreted. */
13374 Log4(("#GP rc=%Rrc\n", rc));
13375 break;
13376 }
13377 }
13378 }
13379 else
13380 rc = VERR_EM_INTERPRETER;
13381
13382 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13383 ("#GP Unexpected rc=%Rrc\n", rc));
13384 return rc;
13385}
13386
13387
13388/**
13389 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13390 * the exception reported in the VMX transient structure back into the VM.
13391 *
13392 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13393 * up-to-date.
13394 */
13395static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13396{
13397 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13398#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13399 Assert(pVCpu->hm.s.fUsingDebugLoop);
13400#endif
13401
13402 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13403 hmR0VmxCheckExitDueToEventDelivery(). */
13404 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13405 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13406 AssertRCReturn(rc, rc);
13407 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13408
13409#ifdef DEBUG_ramshankar
13410 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13411 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13412 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13413#endif
13414
13415 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13416 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13417 return VINF_SUCCESS;
13418}
13419
13420
13421/**
13422 * VM-exit exception handler for \#PF (Page-fault exception).
13423 */
13424static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13425{
13426 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13427 PVM pVM = pVCpu->CTX_SUFF(pVM);
13428 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13429 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13430 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13431 AssertRCReturn(rc, rc);
13432
13433 if (!pVM->hm.s.fNestedPaging)
13434 { /* likely */ }
13435 else
13436 {
13437#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13438 Assert(pVCpu->hm.s.fUsingDebugLoop);
13439#endif
13440 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13441 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13442 {
13443 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13444 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13445 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13446 }
13447 else
13448 {
13449 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13450 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13451 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13452 }
13453 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13454 return rc;
13455 }
13456
13457 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13458 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13459 if (pVmxTransient->fVectoringPF)
13460 {
13461 Assert(pVCpu->hm.s.Event.fPending);
13462 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13463 }
13464
13465 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13466 AssertRCReturn(rc, rc);
13467
13468 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13469 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13470
13471 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13472 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13473 (RTGCPTR)pVmxTransient->uExitQualification);
13474
13475 Log4(("#PF: rc=%Rrc\n", rc));
13476 if (rc == VINF_SUCCESS)
13477 {
13478#if 0
13479 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13480 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13481 * memory? We don't update the whole state here... */
13482 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13483 | HM_CHANGED_GUEST_RSP
13484 | HM_CHANGED_GUEST_RFLAGS
13485 | HM_CHANGED_VMX_GUEST_APIC_STATE);
13486#else
13487 /*
13488 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13489 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13490 */
13491 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13492 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13493#endif
13494 TRPMResetTrap(pVCpu);
13495 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13496 return rc;
13497 }
13498
13499 if (rc == VINF_EM_RAW_GUEST_TRAP)
13500 {
13501 if (!pVmxTransient->fVectoringDoublePF)
13502 {
13503 /* It's a guest page fault and needs to be reflected to the guest. */
13504 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13505 TRPMResetTrap(pVCpu);
13506 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13507 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
13508 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13509 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13510 }
13511 else
13512 {
13513 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13514 TRPMResetTrap(pVCpu);
13515 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13516 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13517 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13518 }
13519
13520 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13521 return VINF_SUCCESS;
13522 }
13523
13524 TRPMResetTrap(pVCpu);
13525 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13526 return rc;
13527}
13528
13529/** @} */
13530
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette