VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 48230

Last change on this file since 48230 was 48230, checked in by vboxsync, 12 years ago

VMM: Propagate errors properly while leaving HM context.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 444.2 KB
Line 
1/* $Id: HMVMXR0.cpp 48230 2013-09-02 14:52:50Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.215389.xyz. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/asm-amd64-x86.h>
23#include <iprt/thread.h>
24#include <iprt/string.h>
25
26#include "HMInternal.h"
27#include <VBox/vmm/vm.h>
28#include "HMVMXR0.h"
29#include <VBox/vmm/pdmapi.h>
30#include <VBox/vmm/dbgf.h>
31#include <VBox/vmm/iem.h>
32#include <VBox/vmm/iom.h>
33#include <VBox/vmm/selm.h>
34#include <VBox/vmm/tm.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#ifdef DEBUG_ramshankar
39#define HMVMX_SAVE_FULL_GUEST_STATE
40#define HMVMX_SYNC_FULL_GUEST_STATE
41#define HMVMX_ALWAYS_CHECK_GUEST_STATE
42#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
43#define HMVMX_ALWAYS_TRAP_PF
44#endif
45
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define HMVMX_IS_64BIT_HOST_MODE() (true)
52typedef RTHCUINTREG HMVMXHCUINTREG;
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54extern "C" uint32_t g_fVMXIs64bitHost;
55# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56typedef uint64_t HMVMXHCUINTREG;
57#else
58# define HMVMX_IS_64BIT_HOST_MODE() (false)
59typedef RTHCUINTREG HMVMXHCUINTREG;
60#endif
61
62/** Use the function table. */
63#define HMVMX_USE_FUNCTION_TABLE
64
65/** Determine which tagged-TLB flush handler to use. */
66#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
67#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
68#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
69#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
70
71/** @name Updated-guest-state flags.
72 * @{ */
73#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94 | HMVMX_UPDATED_GUEST_RSP \
95 | HMVMX_UPDATED_GUEST_RFLAGS \
96 | HMVMX_UPDATED_GUEST_CR0 \
97 | HMVMX_UPDATED_GUEST_CR3 \
98 | HMVMX_UPDATED_GUEST_CR4 \
99 | HMVMX_UPDATED_GUEST_GDTR \
100 | HMVMX_UPDATED_GUEST_IDTR \
101 | HMVMX_UPDATED_GUEST_LDTR \
102 | HMVMX_UPDATED_GUEST_TR \
103 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104 | HMVMX_UPDATED_GUEST_DEBUG \
105 | HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106 | HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112 | HMVMX_UPDATED_GUEST_APIC_STATE)
113/** @} */
114
115/** @name
116 * Flags to skip redundant reads of some common VMCS fields that are not part of
117 * the guest-CPU state but are in the transient structure.
118 */
119#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
125/** @} */
126
127/** @name
128 * States of the VMCS.
129 *
130 * This does not reflect all possible VMCS states but currently only those
131 * needed for maintaining the VMCS consistently even when thread-context hooks
132 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
133 */
134#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
135#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
136#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
137/** @} */
138
139/**
140 * Exception bitmap mask for real-mode guests (real-on-v86).
141 *
142 * We need to intercept all exceptions manually (except #PF). #NM is also
143 * handled separately, see hmR0VmxLoadGuestControlRegs(). #PF need not be
144 * intercepted even in real-mode if we have Nested Paging support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/*******************************************************************************
187* Structures and Typedefs *
188*******************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG uEflags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
203 uint64_t u64LStarMsr;
204 /** The guest's TPR value used for TPR shadowing. */
205 uint8_t u8GuestTpr;
206 /** Alignment. */
207 uint8_t abAlignment0[7];
208
209 /** The basic VM-exit reason. */
210 uint16_t uExitReason;
211 /** Alignment. */
212 uint16_t u16Alignment0;
213 /** The VM-exit interruption error code. */
214 uint32_t uExitIntrErrorCode;
215 /** The VM-exit exit qualification. */
216 uint64_t uExitQualification;
217
218 /** The VM-exit interruption-information field. */
219 uint32_t uExitIntrInfo;
220 /** The VM-exit instruction-length field. */
221 uint32_t cbInstr;
222 /** The VM-exit instruction-information field. */
223 union
224 {
225 /** Plain unsigned int representation. */
226 uint32_t u;
227 /** INS and OUTS information. */
228 struct
229 {
230 uint32_t u6Reserved0 : 6;
231 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
232 uint32_t u3AddrSize : 3;
233 uint32_t u5Reserved1 : 5;
234 /** The segment register (X86_SREG_XXX). */
235 uint32_t iSegReg : 3;
236 uint32_t uReserved2 : 14;
237 } StrIo;
238 } ExitInstrInfo;
239 /** Whether the VM-entry failed or not. */
240 bool fVMEntryFailed;
241 /** Alignment. */
242 uint8_t abAlignment1[3];
243
244 /** The VM-entry interruption-information field. */
245 uint32_t uEntryIntrInfo;
246 /** The VM-entry exception error code field. */
247 uint32_t uEntryXcptErrorCode;
248 /** The VM-entry instruction length field. */
249 uint32_t cbEntryInstr;
250
251 /** IDT-vectoring information field. */
252 uint32_t uIdtVectoringInfo;
253 /** IDT-vectoring error code. */
254 uint32_t uIdtVectoringErrorCode;
255
256 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
257 uint32_t fVmcsFieldsRead;
258 /** Whether TSC-offsetting should be setup before VM-entry. */
259 bool fUpdateTscOffsettingAndPreemptTimer;
260 /** Whether the VM-exit was caused by a page-fault during delivery of a
261 * contributory exception or a page-fault. */
262 bool fVectoringPF;
263} VMXTRANSIENT;
264AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
265AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
266AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
267AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
268/** Pointer to VMX transient state. */
269typedef VMXTRANSIENT *PVMXTRANSIENT;
270
271
272/**
273 * MSR-bitmap read permissions.
274 */
275typedef enum VMXMSREXITREAD
276{
277 /** Reading this MSR causes a VM-exit. */
278 VMXMSREXIT_INTERCEPT_READ = 0xb,
279 /** Reading this MSR does not cause a VM-exit. */
280 VMXMSREXIT_PASSTHRU_READ
281} VMXMSREXITREAD;
282
283/**
284 * MSR-bitmap write permissions.
285 */
286typedef enum VMXMSREXITWRITE
287{
288 /** Writing to this MSR causes a VM-exit. */
289 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
290 /** Writing to this MSR does not cause a VM-exit. */
291 VMXMSREXIT_PASSTHRU_WRITE
292} VMXMSREXITWRITE;
293
294/**
295 * VM-exit handler.
296 *
297 * @returns VBox status code.
298 * @param pVCpu Pointer to the VMCPU.
299 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
300 * out-of-sync. Make sure to update the required
301 * fields before using them.
302 * @param pVmxTransient Pointer to the VMX-transient structure.
303 */
304#ifndef HMVMX_USE_FUNCTION_TABLE
305typedef int FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306#else
307typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308/** Pointer to VM-exit handler. */
309typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
310#endif
311
312
313/*******************************************************************************
314* Internal Functions *
315*******************************************************************************/
316static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
317static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
318static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
319 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
320#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
321static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
322#endif
323#ifndef HMVMX_USE_FUNCTION_TABLE
324DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
325# define HMVMX_EXIT_DECL static int
326#else
327# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
328#endif
329
330/** @name VM-exit handlers.
331 * @{
332 */
333static FNVMEXITHANDLER hmR0VmxExitXcptOrNmi;
334static FNVMEXITHANDLER hmR0VmxExitExtInt;
335static FNVMEXITHANDLER hmR0VmxExitTripleFault;
336static FNVMEXITHANDLER hmR0VmxExitInitSignal;
337static FNVMEXITHANDLER hmR0VmxExitSipi;
338static FNVMEXITHANDLER hmR0VmxExitIoSmi;
339static FNVMEXITHANDLER hmR0VmxExitSmi;
340static FNVMEXITHANDLER hmR0VmxExitIntWindow;
341static FNVMEXITHANDLER hmR0VmxExitNmiWindow;
342static FNVMEXITHANDLER hmR0VmxExitTaskSwitch;
343static FNVMEXITHANDLER hmR0VmxExitCpuid;
344static FNVMEXITHANDLER hmR0VmxExitGetsec;
345static FNVMEXITHANDLER hmR0VmxExitHlt;
346static FNVMEXITHANDLER hmR0VmxExitInvd;
347static FNVMEXITHANDLER hmR0VmxExitInvlpg;
348static FNVMEXITHANDLER hmR0VmxExitRdpmc;
349static FNVMEXITHANDLER hmR0VmxExitRdtsc;
350static FNVMEXITHANDLER hmR0VmxExitRsm;
351static FNVMEXITHANDLER hmR0VmxExitSetPendingXcptUD;
352static FNVMEXITHANDLER hmR0VmxExitMovCRx;
353static FNVMEXITHANDLER hmR0VmxExitMovDRx;
354static FNVMEXITHANDLER hmR0VmxExitIoInstr;
355static FNVMEXITHANDLER hmR0VmxExitRdmsr;
356static FNVMEXITHANDLER hmR0VmxExitWrmsr;
357static FNVMEXITHANDLER hmR0VmxExitErrInvalidGuestState;
358static FNVMEXITHANDLER hmR0VmxExitErrMsrLoad;
359static FNVMEXITHANDLER hmR0VmxExitErrUndefined;
360static FNVMEXITHANDLER hmR0VmxExitMwait;
361static FNVMEXITHANDLER hmR0VmxExitMtf;
362static FNVMEXITHANDLER hmR0VmxExitMonitor;
363static FNVMEXITHANDLER hmR0VmxExitPause;
364static FNVMEXITHANDLER hmR0VmxExitErrMachineCheck;
365static FNVMEXITHANDLER hmR0VmxExitTprBelowThreshold;
366static FNVMEXITHANDLER hmR0VmxExitApicAccess;
367static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
368static FNVMEXITHANDLER hmR0VmxExitXdtrAccess;
369static FNVMEXITHANDLER hmR0VmxExitEptViolation;
370static FNVMEXITHANDLER hmR0VmxExitEptMisconfig;
371static FNVMEXITHANDLER hmR0VmxExitRdtscp;
372static FNVMEXITHANDLER hmR0VmxExitPreemptTimer;
373static FNVMEXITHANDLER hmR0VmxExitWbinvd;
374static FNVMEXITHANDLER hmR0VmxExitXsetbv;
375static FNVMEXITHANDLER hmR0VmxExitRdrand;
376static FNVMEXITHANDLER hmR0VmxExitInvpcid;
377/** @} */
378
379static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
380static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
381static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
382static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
383static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
384static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
385static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
386static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
387
388/*******************************************************************************
389* Global Variables *
390*******************************************************************************/
391#ifdef HMVMX_USE_FUNCTION_TABLE
392
393/**
394 * VMX_EXIT dispatch table.
395 */
396static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
397{
398 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
399 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
400 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
401 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
402 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
403 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
404 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
405 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
406 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
407 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
408 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
409 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
410 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
411 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
412 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
413 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
414 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
415 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
416 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
417 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
418 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
419 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
420 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
421 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
422 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
423 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
424 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
425 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
426 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
427 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
428 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
429 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
430 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
431 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
432 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
433 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
434 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
435 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
436 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
437 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
438 /* 40 UNDEFINED */ hmR0VmxExitPause,
439 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
440 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
441 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
442 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
443 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
444 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
445 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
446 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
447 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
448 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
449 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
450 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
451 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
452 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
453 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
454 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
455 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
456 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
457 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
458};
459#endif /* HMVMX_USE_FUNCTION_TABLE */
460
461#ifdef VBOX_STRICT
462static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
463{
464 /* 0 */ "(Not Used)",
465 /* 1 */ "VMCALL executed in VMX root operation.",
466 /* 2 */ "VMCLEAR with invalid physical address.",
467 /* 3 */ "VMCLEAR with VMXON pointer.",
468 /* 4 */ "VMLAUNCH with non-clear VMCS.",
469 /* 5 */ "VMRESUME with non-launched VMCS.",
470 /* 6 */ "VMRESUME after VMXOFF",
471 /* 7 */ "VM entry with invalid control fields.",
472 /* 8 */ "VM entry with invalid host state fields.",
473 /* 9 */ "VMPTRLD with invalid physical address.",
474 /* 10 */ "VMPTRLD with VMXON pointer.",
475 /* 11 */ "VMPTRLD with incorrect revision identifier.",
476 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
477 /* 13 */ "VMWRITE to read-only VMCS component.",
478 /* 14 */ "(Not Used)",
479 /* 15 */ "VMXON executed in VMX root operation.",
480 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
481 /* 17 */ "VM entry with non-launched executing VMCS.",
482 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
483 /* 19 */ "VMCALL with non-clear VMCS.",
484 /* 20 */ "VMCALL with invalid VM-exit control fields.",
485 /* 21 */ "(Not Used)",
486 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
487 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
488 /* 24 */ "VMCALL with invalid SMM-monitor features.",
489 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
490 /* 26 */ "VM entry with events blocked by MOV SS.",
491 /* 27 */ "(Not Used)",
492 /* 28 */ "Invalid operand to INVEPT/INVVPID."
493};
494#endif /* VBOX_STRICT */
495
496
497
498/**
499 * Updates the VM's last error record. If there was a VMX instruction error,
500 * reads the error data from the VMCS and updates VCPU's last error record as
501 * well.
502 *
503 * @param pVM Pointer to the VM.
504 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
505 * VERR_VMX_UNABLE_TO_START_VM or
506 * VERR_VMX_INVALID_VMCS_FIELD).
507 * @param rc The error code.
508 */
509static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
510{
511 AssertPtr(pVM);
512 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
513 || rc == VERR_VMX_UNABLE_TO_START_VM)
514 {
515 AssertPtrReturnVoid(pVCpu);
516 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
517 }
518 pVM->hm.s.lLastError = rc;
519}
520
521
522/**
523 * Reads the VM-entry interruption-information field from the VMCS into the VMX
524 * transient structure.
525 *
526 * @returns VBox status code.
527 * @param pVmxTransient Pointer to the VMX transient structure.
528 *
529 * @remarks No-long-jump zone!!!
530 */
531DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
532{
533 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
534 AssertRCReturn(rc, rc);
535 return VINF_SUCCESS;
536}
537
538
539/**
540 * Reads the VM-entry exception error code field from the VMCS into
541 * the VMX transient structure.
542 *
543 * @returns VBox status code.
544 * @param pVmxTransient Pointer to the VMX transient structure.
545 *
546 * @remarks No-long-jump zone!!!
547 */
548DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
549{
550 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
551 AssertRCReturn(rc, rc);
552 return VINF_SUCCESS;
553}
554
555
556/**
557 * Reads the VM-entry exception error code field from the VMCS into
558 * the VMX transient structure.
559 *
560 * @returns VBox status code.
561 * @param pVCpu Pointer to the VMCPU.
562 * @param pVmxTransient Pointer to the VMX transient structure.
563 *
564 * @remarks No-long-jump zone!!!
565 */
566DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
567{
568 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
569 AssertRCReturn(rc, rc);
570 return VINF_SUCCESS;
571}
572
573
574/**
575 * Reads the VM-exit interruption-information field from the VMCS into the VMX
576 * transient structure.
577 *
578 * @returns VBox status code.
579 * @param pVCpu Pointer to the VMCPU.
580 * @param pVmxTransient Pointer to the VMX transient structure.
581 */
582DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583{
584 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
585 {
586 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
587 AssertRCReturn(rc, rc);
588 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
589 }
590 return VINF_SUCCESS;
591}
592
593
594/**
595 * Reads the VM-exit interruption error code from the VMCS into the VMX
596 * transient structure.
597 *
598 * @returns VBox status code.
599 * @param pVCpu Pointer to the VMCPU.
600 * @param pVmxTransient Pointer to the VMX transient structure.
601 */
602DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
603{
604 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
605 {
606 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
607 AssertRCReturn(rc, rc);
608 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
609 }
610 return VINF_SUCCESS;
611}
612
613
614/**
615 * Reads the VM-exit instruction length field from the VMCS into the VMX
616 * transient structure.
617 *
618 * @returns VBox status code.
619 * @param pVCpu Pointer to the VMCPU.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction-information field from the VMCS into
636 * the VMX transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu The cross context per CPU structure.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the exit qualification from the VMCS into the VMX transient structure.
656 *
657 * @returns VBox status code.
658 * @param pVCpu Pointer to the VMCPU.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
664 {
665 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the IDT-vectoring information field from the VMCS into the VMX
675 * transient structure.
676 *
677 * @returns VBox status code.
678 * @param pVmxTransient Pointer to the VMX transient structure.
679 *
680 * @remarks No-long-jump zone!!!
681 */
682DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
683{
684 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
685 {
686 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
687 AssertRCReturn(rc, rc);
688 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
689 }
690 return VINF_SUCCESS;
691}
692
693
694/**
695 * Reads the IDT-vectoring error code from the VMCS into the VMX
696 * transient structure.
697 *
698 * @returns VBox status code.
699 * @param pVmxTransient Pointer to the VMX transient structure.
700 */
701DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
702{
703 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
704 {
705 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
706 AssertRCReturn(rc, rc);
707 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
708 }
709 return VINF_SUCCESS;
710}
711
712
713/**
714 * Enters VMX root mode operation on the current CPU.
715 *
716 * @returns VBox status code.
717 * @param pVM Pointer to the VM (optional, can be NULL, after
718 * a resume).
719 * @param HCPhysCpuPage Physical address of the VMXON region.
720 * @param pvCpuPage Pointer to the VMXON region.
721 */
722static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
723{
724 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
725 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
726 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
727
728 if (pVM)
729 {
730 /* Write the VMCS revision dword to the VMXON region. */
731 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
732 }
733
734 /* Enable the VMX bit in CR4 if necessary. */
735 RTCCUINTREG uCr4 = ASMGetCR4();
736 if (!(uCr4 & X86_CR4_VMXE))
737 ASMSetCR4(uCr4 | X86_CR4_VMXE);
738
739 /* Enter VMX root mode. */
740 int rc = VMXEnable(HCPhysCpuPage);
741 if (RT_FAILURE(rc))
742 ASMSetCR4(uCr4);
743
744 return rc;
745}
746
747
748/**
749 * Exits VMX root mode operation on the current CPU.
750 *
751 * @returns VBox status code.
752 */
753static int hmR0VmxLeaveRootMode(void)
754{
755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
756
757 /* If we're for some reason not in VMX root mode, then don't leave it. */
758 RTCCUINTREG uHostCR4 = ASMGetCR4();
759 if (uHostCR4 & X86_CR4_VMXE)
760 {
761 /* Exit VMX root mode and clear the VMX bit in CR4. */
762 VMXDisable();
763 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
764 return VINF_SUCCESS;
765 }
766
767 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
768}
769
770
771/**
772 * Allocates and maps one physically contiguous page. The allocated page is
773 * zero'd out. (Used by various VT-x structures).
774 *
775 * @returns IPRT status code.
776 * @param pMemObj Pointer to the ring-0 memory object.
777 * @param ppVirt Where to store the virtual address of the
778 * allocation.
779 * @param pPhys Where to store the physical address of the
780 * allocation.
781 */
782DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
783{
784 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
785 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
786 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
787
788 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
789 if (RT_FAILURE(rc))
790 return rc;
791 *ppVirt = RTR0MemObjAddress(*pMemObj);
792 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
793 ASMMemZero32(*ppVirt, PAGE_SIZE);
794 return VINF_SUCCESS;
795}
796
797
798/**
799 * Frees and unmaps an allocated physical page.
800 *
801 * @param pMemObj Pointer to the ring-0 memory object.
802 * @param ppVirt Where to re-initialize the virtual address of
803 * allocation as 0.
804 * @param pHCPhys Where to re-initialize the physical address of the
805 * allocation as 0.
806 */
807DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
808{
809 AssertPtr(pMemObj);
810 AssertPtr(ppVirt);
811 AssertPtr(pHCPhys);
812 if (*pMemObj != NIL_RTR0MEMOBJ)
813 {
814 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
815 AssertRC(rc);
816 *pMemObj = NIL_RTR0MEMOBJ;
817 *ppVirt = 0;
818 *pHCPhys = 0;
819 }
820}
821
822
823/**
824 * Worker function to free VT-x related structures.
825 *
826 * @returns IPRT status code.
827 * @param pVM Pointer to the VM.
828 */
829static void hmR0VmxStructsFree(PVM pVM)
830{
831 for (VMCPUID i = 0; i < pVM->cCpus; i++)
832 {
833 PVMCPU pVCpu = &pVM->aCpus[i];
834 AssertPtr(pVCpu);
835
836#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
837 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
838 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
839#endif
840
841 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
842 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
843
844 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
845 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
846 }
847
848 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
849#ifdef VBOX_WITH_CRASHDUMP_MAGIC
850 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
851#endif
852}
853
854
855/**
856 * Worker function to allocate VT-x related VM structures.
857 *
858 * @returns IPRT status code.
859 * @param pVM Pointer to the VM.
860 */
861static int hmR0VmxStructsAlloc(PVM pVM)
862{
863 /*
864 * Initialize members up-front so we can cleanup properly on allocation failure.
865 */
866#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
867 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
868 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
869 pVM->hm.s.vmx.HCPhys##a_Name = 0;
870
871#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
872 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
873 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
874 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
875
876#ifdef VBOX_WITH_CRASHDUMP_MAGIC
877 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
878#endif
879 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
880
881 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
882 for (VMCPUID i = 0; i < pVM->cCpus; i++)
883 {
884 PVMCPU pVCpu = &pVM->aCpus[i];
885 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
886 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
887 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
888#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
889 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
890 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
891#endif
892 }
893#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
894#undef VMXLOCAL_INIT_VM_MEMOBJ
895
896 /*
897 * Allocate all the VT-x structures.
898 */
899 int rc = VINF_SUCCESS;
900#ifdef VBOX_WITH_CRASHDUMP_MAGIC
901 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
902 if (RT_FAILURE(rc))
903 goto cleanup;
904 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
905 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
906#endif
907
908 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
909 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
910 {
911 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
912 &pVM->hm.s.vmx.HCPhysApicAccess);
913 if (RT_FAILURE(rc))
914 goto cleanup;
915 }
916
917 /*
918 * Initialize per-VCPU VT-x structures.
919 */
920 for (VMCPUID i = 0; i < pVM->cCpus; i++)
921 {
922 PVMCPU pVCpu = &pVM->aCpus[i];
923 AssertPtr(pVCpu);
924
925 /* Allocate the VM control structure (VMCS). */
926 AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
927 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 if (RT_FAILURE(rc))
929 goto cleanup;
930
931 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
932 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
933 {
934 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
935 &pVCpu->hm.s.vmx.HCPhysVirtApic);
936 if (RT_FAILURE(rc))
937 goto cleanup;
938 }
939
940 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
941 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
942 {
943 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
944 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
945 if (RT_FAILURE(rc))
946 goto cleanup;
947 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
948 }
949
950#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
951 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
952 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955
956 /* Allocate the VM-exit MSR-load page for the host MSRs. */
957 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
958 if (RT_FAILURE(rc))
959 goto cleanup;
960#endif
961 }
962
963 return VINF_SUCCESS;
964
965cleanup:
966 hmR0VmxStructsFree(pVM);
967 return rc;
968}
969
970
971/**
972 * Does global VT-x initialization (called during module initialization).
973 *
974 * @returns VBox status code.
975 */
976VMMR0DECL(int) VMXR0GlobalInit(void)
977{
978#ifdef HMVMX_USE_FUNCTION_TABLE
979 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
980# ifdef VBOX_STRICT
981 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
982 Assert(g_apfnVMExitHandlers[i]);
983# endif
984#endif
985 return VINF_SUCCESS;
986}
987
988
989/**
990 * Does global VT-x termination (called during module termination).
991 */
992VMMR0DECL(void) VMXR0GlobalTerm()
993{
994 /* Nothing to do currently. */
995}
996
997
998/**
999 * Sets up and activates VT-x on the current CPU.
1000 *
1001 * @returns VBox status code.
1002 * @param pCpu Pointer to the global CPU info struct.
1003 * @param pVM Pointer to the VM (can be NULL after a host resume
1004 * operation).
1005 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1006 * fEnabledByHost is true).
1007 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1008 * @a fEnabledByHost is true).
1009 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1010 * enable VT-x on the host.
1011 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1012 */
1013VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1014 void *pvMsrs)
1015{
1016 AssertReturn(pCpu, VERR_INVALID_PARAMETER);
1017 AssertReturn(pvMsrs, VERR_INVALID_PARAMETER);
1018 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1019
1020 /* Enable VT-x if it's not already enabled by the host. */
1021 if (!fEnabledByHost)
1022 {
1023 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1024 if (RT_FAILURE(rc))
1025 return rc;
1026 }
1027
1028 /*
1029 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1030 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1031 */
1032 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1033 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1034 {
1035 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1036 pCpu->fFlushAsidBeforeUse = false;
1037 }
1038 else
1039 pCpu->fFlushAsidBeforeUse = true;
1040
1041 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1042 ++pCpu->cTlbFlushes;
1043
1044 return VINF_SUCCESS;
1045}
1046
1047
1048/**
1049 * Deactivates VT-x on the current CPU.
1050 *
1051 * @returns VBox status code.
1052 * @param pCpu Pointer to the global CPU info struct.
1053 * @param pvCpuPage Pointer to the VMXON region.
1054 * @param HCPhysCpuPage Physical address of the VMXON region.
1055 *
1056 * @remarks This function should never be called when SUPR0EnableVTx() or
1057 * similar was used to enable VT-x on the host.
1058 */
1059VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1060{
1061 NOREF(pCpu);
1062 NOREF(pvCpuPage);
1063 NOREF(HCPhysCpuPage);
1064
1065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1066 return hmR0VmxLeaveRootMode();
1067}
1068
1069
1070/**
1071 * Sets the permission bits for the specified MSR in the MSR bitmap.
1072 *
1073 * @param pVCpu Pointer to the VMCPU.
1074 * @param uMSR The MSR value.
1075 * @param enmRead Whether reading this MSR causes a VM-exit.
1076 * @param enmWrite Whether writing this MSR causes a VM-exit.
1077 */
1078static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1079{
1080 int32_t iBit;
1081 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1082
1083 /*
1084 * Layout:
1085 * 0x000 - 0x3ff - Low MSR read bits
1086 * 0x400 - 0x7ff - High MSR read bits
1087 * 0x800 - 0xbff - Low MSR write bits
1088 * 0xc00 - 0xfff - High MSR write bits
1089 */
1090 if (uMsr <= 0x00001FFF)
1091 iBit = uMsr;
1092 else if ( uMsr >= 0xC0000000
1093 && uMsr <= 0xC0001FFF)
1094 {
1095 iBit = (uMsr - 0xC0000000);
1096 pbMsrBitmap += 0x400;
1097 }
1098 else
1099 {
1100 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1101 return;
1102 }
1103
1104 Assert(iBit <= 0x1fff);
1105 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1106 ASMBitSet(pbMsrBitmap, iBit);
1107 else
1108 ASMBitClear(pbMsrBitmap, iBit);
1109
1110 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1111 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1112 else
1113 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1114}
1115
1116
1117/**
1118 * Flushes the TLB using EPT.
1119 *
1120 * @returns VBox status code.
1121 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1122 * enmFlush).
1123 * @param enmFlush Type of flush.
1124 *
1125 * @remarks Caller is responsible for making sure this function is called only
1126 * when NestedPaging is supported and providing @a enmFlush that is
1127 * supported by the CPU.
1128 */
1129static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1130{
1131 uint64_t au64Descriptor[2];
1132 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1133 au64Descriptor[0] = 0;
1134 else
1135 {
1136 Assert(pVCpu);
1137 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1138 }
1139 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1140
1141 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1142 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1143 rc));
1144 if ( RT_SUCCESS(rc)
1145 && pVCpu)
1146 {
1147 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1148 }
1149}
1150
1151
1152/**
1153 * Flushes the TLB using VPID.
1154 *
1155 * @returns VBox status code.
1156 * @param pVM Pointer to the VM.
1157 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1158 * enmFlush).
1159 * @param enmFlush Type of flush.
1160 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1161 * on @a enmFlush).
1162 */
1163static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1164{
1165 AssertPtr(pVM);
1166 Assert(pVM->hm.s.vmx.fVpid);
1167
1168 uint64_t au64Descriptor[2];
1169 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1170 {
1171 au64Descriptor[0] = 0;
1172 au64Descriptor[1] = 0;
1173 }
1174 else
1175 {
1176 AssertPtr(pVCpu);
1177 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1178 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1179 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1180 au64Descriptor[1] = GCPtr;
1181 }
1182
1183 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1184 AssertMsg(rc == VINF_SUCCESS,
1185 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1186 if ( RT_SUCCESS(rc)
1187 && pVCpu)
1188 {
1189 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1190 }
1191}
1192
1193
1194/**
1195 * Invalidates a guest page by guest virtual address. Only relevant for
1196 * EPT/VPID, otherwise there is nothing really to invalidate.
1197 *
1198 * @returns VBox status code.
1199 * @param pVM Pointer to the VM.
1200 * @param pVCpu Pointer to the VMCPU.
1201 * @param GCVirt Guest virtual address of the page to invalidate.
1202 */
1203VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1204{
1205 AssertPtr(pVM);
1206 AssertPtr(pVCpu);
1207 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1208
1209 bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1210 if (!fFlushPending)
1211 {
1212 /*
1213 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1214 * See @bugref{6043} and @bugref{6177}.
1215 *
1216 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1217 * function maybe called in a loop with individual addresses.
1218 */
1219 if (pVM->hm.s.vmx.fVpid)
1220 {
1221 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1222 {
1223 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1224 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1225 }
1226 else
1227 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1228 }
1229 else if (pVM->hm.s.fNestedPaging)
1230 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1231 }
1232
1233 return VINF_SUCCESS;
1234}
1235
1236
1237/**
1238 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1239 * otherwise there is nothing really to invalidate.
1240 *
1241 * @returns VBox status code.
1242 * @param pVM Pointer to the VM.
1243 * @param pVCpu Pointer to the VMCPU.
1244 * @param GCPhys Guest physical address of the page to invalidate.
1245 */
1246VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1247{
1248 LogFlowFunc(("%RGp\n", GCPhys));
1249
1250 /*
1251 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1252 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1253 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1254 */
1255 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1256 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1257 return VINF_SUCCESS;
1258}
1259
1260
1261/**
1262 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1263 * case where neither EPT nor VPID is supported by the CPU.
1264 *
1265 * @param pVM Pointer to the VM.
1266 * @param pVCpu Pointer to the VMCPU.
1267 *
1268 * @remarks Called with interrupts disabled.
1269 */
1270static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1271{
1272 NOREF(pVM);
1273 AssertPtr(pVCpu);
1274 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1275 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1276
1277 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1278 AssertPtr(pCpu);
1279
1280 pVCpu->hm.s.TlbShootdown.cPages = 0;
1281 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1282 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1283 pVCpu->hm.s.fForceTLBFlush = false;
1284 return;
1285}
1286
1287
1288/**
1289 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1290 *
1291 * @param pVM Pointer to the VM.
1292 * @param pVCpu Pointer to the VMCPU.
1293 * @remarks All references to "ASID" in this function pertains to "VPID" in
1294 * Intel's nomenclature. The reason is, to avoid confusion in compare
1295 * statements since the host-CPU copies are named "ASID".
1296 *
1297 * @remarks Called with interrupts disabled.
1298 */
1299static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1300{
1301#ifdef VBOX_WITH_STATISTICS
1302 bool fTlbFlushed = false;
1303# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1304# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1305 if (!fTlbFlushed) \
1306 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1307 } while (0)
1308#else
1309# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1310# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1311#endif
1312
1313 AssertPtr(pVM);
1314 AssertPtr(pVCpu);
1315 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1316 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1317 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1318
1319 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1320 AssertPtr(pCpu);
1321
1322 /*
1323 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1324 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1325 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1326 */
1327 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1328 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1329 {
1330 ++pCpu->uCurrentAsid;
1331 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1332 {
1333 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1334 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1335 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1336 }
1337
1338 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1339 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1340 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1341
1342 /*
1343 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1344 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1345 */
1346 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1347 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1348 HMVMX_SET_TAGGED_TLB_FLUSHED();
1349 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1350 }
1351
1352 /* Check for explicit TLB shootdowns. */
1353 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1354 {
1355 /*
1356 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1357 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1358 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1359 * but not guest-physical mappings.
1360 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1361 */
1362 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1363 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1364 HMVMX_SET_TAGGED_TLB_FLUSHED();
1365 }
1366
1367 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1368 * not be executed. See hmQueueInvlPage() where it is commented
1369 * out. Support individual entry flushing someday. */
1370 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1371 {
1372 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1373
1374 /*
1375 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1376 * as supported by the CPU.
1377 */
1378 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1379 {
1380 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1381 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1382 }
1383 else
1384 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1385
1386 HMVMX_SET_TAGGED_TLB_FLUSHED();
1387 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1388 }
1389
1390 pVCpu->hm.s.TlbShootdown.cPages = 0;
1391 pVCpu->hm.s.fForceTLBFlush = false;
1392
1393 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1394
1395 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1396 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1397 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1398 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1399 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1400 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1401 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1402 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1403
1404 /* Update VMCS with the VPID. */
1405 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1406 AssertRC(rc);
1407
1408#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1409}
1410
1411
1412/**
1413 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1414 *
1415 * @returns VBox status code.
1416 * @param pVM Pointer to the VM.
1417 * @param pVCpu Pointer to the VMCPU.
1418 *
1419 * @remarks Called with interrupts disabled.
1420 */
1421static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1422{
1423 AssertPtr(pVM);
1424 AssertPtr(pVCpu);
1425 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1426 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1427
1428 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1429 AssertPtr(pCpu);
1430
1431 /*
1432 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1433 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1434 */
1435 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1436 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1437 {
1438 pVCpu->hm.s.fForceTLBFlush = true;
1439 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1440 }
1441
1442 /* Check for explicit TLB shootdown flushes. */
1443 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1444 {
1445 pVCpu->hm.s.fForceTLBFlush = true;
1446 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1447 }
1448
1449 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1450 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1451
1452 if (pVCpu->hm.s.fForceTLBFlush)
1453 {
1454 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1455 pVCpu->hm.s.fForceTLBFlush = false;
1456 }
1457 else
1458 {
1459 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1460 * not be executed. See hmQueueInvlPage() where it is commented
1461 * out. Support individual entry flushing someday. */
1462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1463 {
1464 /* We cannot flush individual entries without VPID support. Flush using EPT. */
1465 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1466 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1467 }
1468 else
1469 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1470 }
1471
1472 pVCpu->hm.s.TlbShootdown.cPages = 0;
1473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1474}
1475
1476
1477/**
1478 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
1479 *
1480 * @returns VBox status code.
1481 * @param pVM Pointer to the VM.
1482 * @param pVCpu Pointer to the VMCPU.
1483 *
1484 * @remarks Called with interrupts disabled.
1485 */
1486static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1487{
1488 AssertPtr(pVM);
1489 AssertPtr(pVCpu);
1490 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1491 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1492
1493 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
1494
1495 /*
1496 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1497 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1498 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1499 */
1500 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1501 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1502 {
1503 pVCpu->hm.s.fForceTLBFlush = true;
1504 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1505 }
1506
1507 /* Check for explicit TLB shootdown flushes. */
1508 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1509 {
1510 /*
1511 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1512 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1513 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1514 */
1515 pVCpu->hm.s.fForceTLBFlush = true;
1516 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1517 }
1518
1519 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1520 if (pVCpu->hm.s.fForceTLBFlush)
1521 {
1522 ++pCpu->uCurrentAsid;
1523 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1524 {
1525 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
1526 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1527 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1528 }
1529
1530 pVCpu->hm.s.fForceTLBFlush = false;
1531 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1532 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1533 if (pCpu->fFlushAsidBeforeUse)
1534 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1535 }
1536 else
1537 {
1538 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1539 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1540 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1541 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1542
1543 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1544 * not be executed. See hmQueueInvlPage() where it is commented
1545 * out. Support individual entry flushing someday. */
1546 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1547 {
1548 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1549 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1550 {
1551 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1552 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1553 }
1554 else
1555 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1556 }
1557 else
1558 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1559 }
1560
1561 pVCpu->hm.s.TlbShootdown.cPages = 0;
1562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1563
1564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1565 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1567 ("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1569 ("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1570
1571 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Flushes the guest TLB entry based on CPU capabilities.
1578 *
1579 * @param pVCpu Pointer to the VMCPU.
1580 */
1581DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1582{
1583 PVM pVM = pVCpu->CTX_SUFF(pVM);
1584 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1585 {
1586 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1587 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1588 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1589 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1590 default:
1591 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1592 break;
1593 }
1594}
1595
1596
1597/**
1598 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1599 * TLB entries from the host TLB before VM-entry.
1600 *
1601 * @returns VBox status code.
1602 * @param pVM Pointer to the VM.
1603 */
1604static int hmR0VmxSetupTaggedTlb(PVM pVM)
1605{
1606 /*
1607 * Determine optimal flush type for Nested Paging.
1608 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1609 * guest execution (see hmR3InitFinalizeR0()).
1610 */
1611 if (pVM->hm.s.fNestedPaging)
1612 {
1613 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1614 {
1615 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1616 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1617 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1618 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1619 else
1620 {
1621 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1622 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1623 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624 }
1625
1626 /* Make sure the write-back cacheable memory type for EPT is supported. */
1627 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1628 {
1629 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
1630 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1631 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1632 }
1633 }
1634 else
1635 {
1636 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1637 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1639 }
1640 }
1641
1642 /*
1643 * Determine optimal flush type for VPID.
1644 */
1645 if (pVM->hm.s.vmx.fVpid)
1646 {
1647 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1648 {
1649 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1650 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1651 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1652 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1653 else
1654 {
1655 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1656 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1657 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1658 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1659 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1660 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1661 pVM->hm.s.vmx.fVpid = false;
1662 }
1663 }
1664 else
1665 {
1666 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1667 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1668 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1669 pVM->hm.s.vmx.fVpid = false;
1670 }
1671 }
1672
1673 /*
1674 * Setup the handler for flushing tagged-TLBs.
1675 */
1676 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1677 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1678 else if (pVM->hm.s.fNestedPaging)
1679 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1680 else if (pVM->hm.s.vmx.fVpid)
1681 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1682 else
1683 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1684 return VINF_SUCCESS;
1685}
1686
1687
1688/**
1689 * Sets up pin-based VM-execution controls in the VMCS.
1690 *
1691 * @returns VBox status code.
1692 * @param pVM Pointer to the VM.
1693 * @param pVCpu Pointer to the VMCPU.
1694 */
1695static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1696{
1697 AssertPtr(pVM);
1698 AssertPtr(pVCpu);
1699
1700 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
1701 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
1702
1703 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1704 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1705 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1706
1707 /* Enable the VMX preemption timer. */
1708 if (pVM->hm.s.vmx.fUsePreemptTimer)
1709 {
1710 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1711 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1712 }
1713
1714 if ((val & zap) != val)
1715 {
1716 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1717 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
1718 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
1719 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1720 }
1721
1722 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1723 AssertRCReturn(rc, rc);
1724
1725 /* Update VCPU with the currently set pin-based VM-execution controls. */
1726 pVCpu->hm.s.vmx.u32PinCtls = val;
1727 return rc;
1728}
1729
1730
1731/**
1732 * Sets up processor-based VM-execution controls in the VMCS.
1733 *
1734 * @returns VBox status code.
1735 * @param pVM Pointer to the VM.
1736 * @param pVMCPU Pointer to the VMCPU.
1737 */
1738static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1739{
1740 AssertPtr(pVM);
1741 AssertPtr(pVCpu);
1742
1743 int rc = VERR_INTERNAL_ERROR_5;
1744 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1745 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1746
1747 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1748 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1749 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1750 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1751 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1752 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1753 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1754
1755 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1756 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1757 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1758 {
1759 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1760 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
1761 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1762 }
1763
1764 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1765 if (!pVM->hm.s.fNestedPaging)
1766 {
1767 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1768 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1769 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1770 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1771 }
1772
1773 /* Use TPR shadowing if supported by the CPU. */
1774 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1775 {
1776 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1777 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1778 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1779 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1780 AssertRCReturn(rc, rc);
1781
1782 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1783 /* CR8 writes causes a VM-exit based on TPR threshold. */
1784 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1785 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1786 }
1787 else
1788 {
1789 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1790 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1791 }
1792
1793 /* Use MSR-bitmaps if supported by the CPU. */
1794 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1795 {
1796 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1797
1798 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1799 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1800 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1801 AssertRCReturn(rc, rc);
1802
1803 /*
1804 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1805 * automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1806 */
1807 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1808 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1809 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1810 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1811 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1812 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1813 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1814 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1815 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1816 }
1817
1818 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1819 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1820 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1821
1822 if ((val & zap) != val)
1823 {
1824 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1825 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
1826 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
1827 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1828 }
1829
1830 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1831 AssertRCReturn(rc, rc);
1832
1833 /* Update VCPU with the currently set processor-based VM-execution controls. */
1834 pVCpu->hm.s.vmx.u32ProcCtls = val;
1835
1836 /*
1837 * Secondary processor-based VM-execution controls.
1838 */
1839 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1840 {
1841 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1842 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1843
1844 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1845 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1846
1847 if (pVM->hm.s.fNestedPaging)
1848 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1849 else
1850 {
1851 /*
1852 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1853 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1854 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1855 */
1856 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1857 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1858 }
1859
1860 if (pVM->hm.s.vmx.fVpid)
1861 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1862
1863 if (pVM->hm.s.vmx.fUnrestrictedGuest)
1864 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1865
1866 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1867 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1868 * done dynamically. */
1869 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1870 {
1871 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1872 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1873 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1874 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1875 AssertRCReturn(rc, rc);
1876 }
1877
1878 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1879 {
1880 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1881 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1882 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1883 }
1884
1885 if ((val & zap) != val)
1886 {
1887 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1888 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
1889 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1890 }
1891
1892 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1893 AssertRCReturn(rc, rc);
1894
1895 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1896 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1897 }
1898 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
1899 {
1900 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
1901 "available\n"));
1902 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1903 }
1904
1905 return VINF_SUCCESS;
1906}
1907
1908
1909/**
1910 * Sets up miscellaneous (everything other than Pin & Processor-based
1911 * VM-execution) control fields in the VMCS.
1912 *
1913 * @returns VBox status code.
1914 * @param pVM Pointer to the VM.
1915 * @param pVCpu Pointer to the VMCPU.
1916 */
1917static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1918{
1919 AssertPtr(pVM);
1920 AssertPtr(pVCpu);
1921
1922 int rc = VERR_GENERAL_FAILURE;
1923
1924 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1925#if 0
1926 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1927 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1928 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1929
1930 /*
1931 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1932 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1933 * We thus use the exception bitmap to control it rather than use both.
1934 */
1935 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1936 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1937
1938 /** @todo Explore possibility of using IO-bitmaps. */
1939 /* All IO & IOIO instructions cause VM-exits. */
1940 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1941 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1942
1943 /* Initialize the MSR-bitmap area. */
1944 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1945 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1946 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1947#endif
1948
1949#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1950 /* Setup MSR autoloading/storing. */
1951 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1952 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1953 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1954 AssertRCReturn(rc, rc);
1955 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1956 AssertRCReturn(rc, rc);
1957
1958 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1959 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1960 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1961 AssertRCReturn(rc, rc);
1962#endif
1963
1964 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1965 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1966 AssertRCReturn(rc, rc);
1967
1968 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
1969#if 0
1970 /* Setup debug controls */
1971 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1972 AssertRCReturn(rc, rc);
1973 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1974 AssertRCReturn(rc, rc);
1975#endif
1976
1977 return rc;
1978}
1979
1980
1981/**
1982 * Sets up the initial exception bitmap in the VMCS based on static conditions
1983 * (i.e. conditions that cannot ever change at runtime).
1984 *
1985 * @returns VBox status code.
1986 * @param pVM Pointer to the VM.
1987 * @param pVCpu Pointer to the VMCPU.
1988 */
1989static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1990{
1991 AssertPtr(pVM);
1992 AssertPtr(pVCpu);
1993
1994 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1995
1996 uint32_t u32XcptBitmap = 0;
1997
1998 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1999 if (!pVM->hm.s.fNestedPaging)
2000 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2001
2002 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2003 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2004 AssertRCReturn(rc, rc);
2005 return rc;
2006}
2007
2008
2009/**
2010 * Sets up the initial guest-state mask. The guest-state mask is consulted
2011 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2012 * for the nested virtualization case (as it would cause a VM-exit).
2013 *
2014 * @param pVCpu Pointer to the VMCPU.
2015 */
2016static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2017{
2018 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2019 pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
2020 return VINF_SUCCESS;
2021}
2022
2023
2024/**
2025 * Does per-VM VT-x initialization.
2026 *
2027 * @returns VBox status code.
2028 * @param pVM Pointer to the VM.
2029 */
2030VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2031{
2032 LogFlowFunc(("pVM=%p\n", pVM));
2033
2034 int rc = hmR0VmxStructsAlloc(pVM);
2035 if (RT_FAILURE(rc))
2036 {
2037 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2038 return rc;
2039 }
2040
2041 return VINF_SUCCESS;
2042}
2043
2044
2045/**
2046 * Does per-VM VT-x termination.
2047 *
2048 * @returns VBox status code.
2049 * @param pVM Pointer to the VM.
2050 */
2051VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2052{
2053 LogFlowFunc(("pVM=%p\n", pVM));
2054
2055#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2056 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2057 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2058#endif
2059 hmR0VmxStructsFree(pVM);
2060 return VINF_SUCCESS;
2061}
2062
2063
2064/**
2065 * Sets up the VM for execution under VT-x.
2066 * This function is only called once per-VM during initialization.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM Pointer to the VM.
2070 */
2071VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2072{
2073 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2074 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2075
2076 LogFlowFunc(("pVM=%p\n", pVM));
2077
2078 /*
2079 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2080 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2081 */
2082 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2083 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2084 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2085 || !pVM->hm.s.vmx.pRealModeTSS))
2086 {
2087 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2088 return VERR_INTERNAL_ERROR;
2089 }
2090
2091#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092 /*
2093 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2094 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2095 */
2096 if ( pVM->hm.s.fAllow64BitGuests
2097 && !HMVMX_IS_64BIT_HOST_MODE())
2098 {
2099 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2100 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2101 }
2102#endif
2103
2104 /* Initialize these always, see hmR3InitFinalizeR0().*/
2105 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2106 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2107
2108 /* Setup the tagged-TLB flush handlers. */
2109 int rc = hmR0VmxSetupTaggedTlb(pVM);
2110 if (RT_FAILURE(rc))
2111 {
2112 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2113 return rc;
2114 }
2115
2116 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2117 {
2118 PVMCPU pVCpu = &pVM->aCpus[i];
2119 AssertPtr(pVCpu);
2120 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2121
2122 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2123 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2124
2125 /* Set revision dword at the beginning of the VMCS structure. */
2126 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2127
2128 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2129 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2130 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2131 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2132
2133 /* Load this VMCS as the current VMCS. */
2134 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2135 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2136 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2137
2138 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2139 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2140 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2141
2142 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2143 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2144 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2145
2146 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2147 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2148 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2149
2150 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2151 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2152 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2153
2154 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2155 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2156 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2157
2158#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2159 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2160 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2161 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2162#endif
2163
2164 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2165 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2166 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2167 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2168
2169 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2170
2171 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2172 }
2173
2174 return VINF_SUCCESS;
2175}
2176
2177
2178/**
2179 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2180 * the VMCS.
2181 *
2182 * @returns VBox status code.
2183 * @param pVM Pointer to the VM.
2184 * @param pVCpu Pointer to the VMCPU.
2185 */
2186DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2187{
2188 RTCCUINTREG uReg = ASMGetCR0();
2189 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2190 AssertRCReturn(rc, rc);
2191
2192#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2193 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2194 if (HMVMX_IS_64BIT_HOST_MODE())
2195 {
2196 uint64_t uRegCR3 = HMR0Get64bitCR3();
2197 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2198 }
2199 else
2200#endif
2201 {
2202 uReg = ASMGetCR3();
2203 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2204 }
2205 AssertRCReturn(rc, rc);
2206
2207 uReg = ASMGetCR4();
2208 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2209 AssertRCReturn(rc, rc);
2210 return rc;
2211}
2212
2213
2214/**
2215 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2216 * the host-state area in the VMCS.
2217 *
2218 * @returns VBox status code.
2219 * @param pVM Pointer to the VM.
2220 * @param pVCpu Pointer to the VMCPU.
2221 */
2222DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2223{
2224 int rc = VERR_INTERNAL_ERROR_5;
2225 RTSEL uSelDS = 0;
2226 RTSEL uSelES = 0;
2227 RTSEL uSelFS = 0;
2228 RTSEL uSelGS = 0;
2229 RTSEL uSelTR = 0;
2230
2231 /*
2232 * Host DS, ES, FS and GS segment registers.
2233 */
2234#if HC_ARCH_BITS == 64
2235 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2236 uSelDS = ASMGetDS();
2237 uSelES = ASMGetES();
2238 uSelFS = ASMGetFS();
2239 uSelGS = ASMGetGS();
2240#endif
2241
2242 /*
2243 * Host CS and SS segment registers.
2244 */
2245 RTSEL uSelCS;
2246 RTSEL uSelSS;
2247#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2248 if (HMVMX_IS_64BIT_HOST_MODE())
2249 {
2250 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2251 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2252 }
2253 else
2254 {
2255 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2256 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2257 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2258 }
2259#else
2260 uSelCS = ASMGetCS();
2261 uSelSS = ASMGetSS();
2262#endif
2263
2264 /*
2265 * Host TR segment register.
2266 */
2267 uSelTR = ASMGetTR();
2268
2269#if HC_ARCH_BITS == 64
2270 /*
2271 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2272 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2273 */
2274 if (uSelDS & (X86_SEL_RPL | X86_SEL_LDT))
2275 {
2276 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_DS;
2277 pVCpu->hm.s.vmx.RestoreHost.uHostSelDS = uSelDS;
2278 uSelDS = 0;
2279 }
2280 if (uSelES & (X86_SEL_RPL | X86_SEL_LDT))
2281 {
2282 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_ES;
2283 pVCpu->hm.s.vmx.RestoreHost.uHostSelES = uSelES;
2284 uSelES = 0;
2285 }
2286 if (uSelFS & (X86_SEL_RPL | X86_SEL_LDT))
2287 {
2288 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_FS;
2289 pVCpu->hm.s.vmx.RestoreHost.uHostSelFS = uSelFS;
2290 uSelFS = 0;
2291 }
2292 if (uSelGS & (X86_SEL_RPL | X86_SEL_LDT))
2293 {
2294 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_GS;
2295 pVCpu->hm.s.vmx.RestoreHost.uHostSelGS = uSelGS;
2296 uSelGS = 0;
2297 }
2298#endif
2299
2300 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2301 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2302 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2303 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2304 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2305 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2306 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2307 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2308 Assert(uSelCS);
2309 Assert(uSelTR);
2310
2311 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2312#if 0
2313 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2314 Assert(uSelSS != 0);
2315#endif
2316
2317 /* Write these host selector fields into the host-state area in the VMCS. */
2318 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2319 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2320#if HC_ARCH_BITS == 64
2321 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2322 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2323 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2324 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2325#endif
2326 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2327
2328 /*
2329 * Host GDTR and IDTR.
2330 */
2331 RTGDTR Gdtr;
2332 RT_ZERO(Gdtr);
2333#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2334 if (HMVMX_IS_64BIT_HOST_MODE())
2335 {
2336 X86XDTR64 Gdtr64;
2337 X86XDTR64 Idtr64;
2338 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2339 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2340 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2341
2342 Gdtr.cbGdt = Gdtr64.cb;
2343 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2344 }
2345 else
2346#endif
2347 {
2348 RTIDTR Idtr;
2349 ASMGetGDTR(&Gdtr);
2350 ASMGetIDTR(&Idtr);
2351 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2352 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2353
2354#if HC_ARCH_BITS == 64
2355 /*
2356 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2357 * maximum limit (0xffff) on every VM-exit.
2358 */
2359 if (Gdtr.cbGdt != 0xffff)
2360 {
2361 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2362 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
2363 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2364 }
2365
2366 /*
2367 * IDT limit is practically 0xfff. Therefore if the host has the limit as 0xfff, VT-x bloating the limit to 0xffff
2368 * is not a problem as it's not possible to get at them anyway. See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2369 * Intel spec. 6.2 "Exception and Interrupt Vectors".
2370 */
2371 if (Idtr.cbIdt < 0x0fff)
2372 {
2373 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
2374 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
2375 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
2376 }
2377#endif
2378 }
2379
2380 /*
2381 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2382 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2383 */
2384 if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2385 {
2386 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2387 return VERR_VMX_INVALID_HOST_STATE;
2388 }
2389
2390 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2391#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2392 if (HMVMX_IS_64BIT_HOST_MODE())
2393 {
2394 /* We need the 64-bit TR base for hybrid darwin. */
2395 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2396 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2397 }
2398 else
2399#endif
2400 {
2401 uintptr_t uTRBase;
2402#if HC_ARCH_BITS == 64
2403 uTRBase = X86DESC64_BASE(pDesc);
2404
2405 /*
2406 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
2407 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
2408 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
2409 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
2410 *
2411 * [1] See Intel spec. 3.5 "System Descriptor Types".
2412 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
2413 */
2414 Assert(pDesc->System.u4Type == 11);
2415 if ( pDesc->System.u16LimitLow != 0x67
2416 || pDesc->System.u4LimitHigh)
2417 {
2418 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
2419 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
2420
2421 /* Store the GDTR here as we need it while restoring TR. */
2422 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
2423 }
2424#else
2425 uTRBase = X86DESC_BASE(pDesc);
2426#endif
2427 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2428 }
2429 AssertRCReturn(rc, rc);
2430
2431 /*
2432 * Host FS base and GS base.
2433 */
2434#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2435 if (HMVMX_IS_64BIT_HOST_MODE())
2436 {
2437 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2438 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2439 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
2440 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
2441
2442# if HC_ARCH_BITS == 64
2443 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
2444 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
2445 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
2446 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
2447 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
2448# endif
2449 }
2450#endif
2451 return rc;
2452}
2453
2454
2455/**
2456 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2457 * host-state area of the VMCS. Theses MSRs will be automatically restored on
2458 * the host after every successful VM exit.
2459 *
2460 * @returns VBox status code.
2461 * @param pVM Pointer to the VM.
2462 * @param pVCpu Pointer to the VMCPU.
2463 */
2464DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2465{
2466 AssertPtr(pVCpu);
2467 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2468
2469 int rc = VINF_SUCCESS;
2470#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2471 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
2472 uint32_t cHostMsrs = 0;
2473 uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2474
2475 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2476 {
2477 uint64_t u64HostEfer = ASMRdMsr(MSR_K6_EFER);
2478
2479# if HC_ARCH_BITS == 64
2480 /* Paranoia. 64-bit code requires these bits to be set always. */
2481 Assert((u64HostEfer & (MSR_K6_EFER_LMA | MSR_K6_EFER_LME)) == (MSR_K6_EFER_LMA | MSR_K6_EFER_LME));
2482
2483 /*
2484 * We currently do not save/restore host EFER, we just make sure it doesn't get modified by VT-x operation.
2485 * All guest accesses (read, write) on EFER cause VM-exits. If we are to conditionally load the guest EFER for
2486 * some reason (e.g. allow transparent reads) we would activate the code below.
2487 */
2488# if 0
2489 /* All our supported 64-bit host platforms must have NXE bit set. Otherwise we can change the below code to save EFER. */
2490 Assert(u64HostEfer & (MSR_K6_EFER_NXE));
2491 /* The SCE bit is only applicable in 64-bit mode. Save EFER if it doesn't match what the guest has.
2492 See Intel spec. 30.10.4.3 "Handling the SYSCALL and SYSRET Instructions". */
2493 if (CPUMIsGuestInLongMode(pVCpu))
2494 {
2495 uint64_t u64GuestEfer;
2496 rc = CPUMQueryGuestMsr(pVCpu, MSR_K6_EFER, &u64GuestEfer);
2497 AssertRC(rc);
2498
2499 if ((u64HostEfer & MSR_K6_EFER_SCE) != (u64GuestEfer & MSR_K6_EFER_SCE))
2500 {
2501 pHostMsr->u32Msr = MSR_K6_EFER;
2502 pHostMsr->u32Reserved = 0;
2503 pHostMsr->u64Value = u64HostEfer;
2504 pHostMsr++; cHostMsrs++;
2505 }
2506 }
2507# endif
2508# else /* HC_ARCH_BITS != 64 */
2509 pHostMsr->u32Msr = MSR_K6_EFER;
2510 pHostMsr->u32Reserved = 0;
2511# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2512 if (CPUMIsGuestInLongMode(pVCpu))
2513 {
2514 /* Must match the EFER value in our 64 bits switcher. */
2515 pHostMsr->u64Value = u64HostEfer | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
2516 }
2517 else
2518# endif
2519 pHostMsr->u64Value = u64HostEfer;
2520 pHostMsr++; cHostMsrs++;
2521# endif /* HC_ARCH_BITS == 64 */
2522 }
2523
2524# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2525 if (HMVMX_IS_64BIT_HOST_MODE())
2526 {
2527 pHostMsr->u32Msr = MSR_K6_STAR;
2528 pHostMsr->u32Reserved = 0;
2529 pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2530 pHostMsr++; cHostMsrs++;
2531 pHostMsr->u32Msr = MSR_K8_LSTAR;
2532 pHostMsr->u32Reserved = 0;
2533 pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2534 pHostMsr++; cHostMsrs++;
2535 pHostMsr->u32Msr = MSR_K8_SF_MASK;
2536 pHostMsr->u32Reserved = 0;
2537 pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2538 pHostMsr++; cHostMsrs++;
2539 pHostMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
2540 pHostMsr->u32Reserved = 0;
2541 pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2542 pHostMsr++; cHostMsrs++;
2543 }
2544# endif
2545
2546 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2547 if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)))
2548 {
2549 LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc)));
2550 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_HOST_MSR_STORAGE;
2551 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2552 }
2553
2554 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2555#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2556
2557 /*
2558 * Host Sysenter MSRs.
2559 */
2560 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2561 AssertRCReturn(rc, rc);
2562#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2563 if (HMVMX_IS_64BIT_HOST_MODE())
2564 {
2565 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2566 AssertRCReturn(rc, rc);
2567 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2568 }
2569 else
2570 {
2571 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2572 AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2574 }
2575#elif HC_ARCH_BITS == 32
2576 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2577 AssertRCReturn(rc, rc);
2578 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2579#else
2580 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2581 AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2583#endif
2584 AssertRCReturn(rc, rc);
2585
2586 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2587 * hmR0VmxSetupExitCtls() !! */
2588 return rc;
2589}
2590
2591
2592/**
2593 * Sets up VM-entry controls in the VMCS. These controls can affect things done
2594 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2595 * controls".
2596 *
2597 * @returns VBox status code.
2598 * @param pVCpu Pointer to the VMCPU.
2599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2600 * out-of-sync. Make sure to update the required fields
2601 * before using them.
2602 *
2603 * @remarks No-long-jump zone!!!
2604 */
2605DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2606{
2607 int rc = VINF_SUCCESS;
2608 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2609 {
2610 PVM pVM = pVCpu->CTX_SUFF(pVM);
2611 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2612 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2613
2614 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2615 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2616
2617 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2618 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2619 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2620 else
2621 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2622
2623 /*
2624 * The following should not be set (since we're not in SMM mode):
2625 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2626 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2627 */
2628
2629 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2630 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2631 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2632
2633 if ((val & zap) != val)
2634 {
2635 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2636 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
2637 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
2638 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2639 }
2640
2641 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2642 AssertRCReturn(rc, rc);
2643
2644 /* Update VCPU with the currently set VM-exit controls. */
2645 pVCpu->hm.s.vmx.u32EntryCtls = val;
2646 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2647 }
2648 return rc;
2649}
2650
2651
2652/**
2653 * Sets up the VM-exit controls in the VMCS.
2654 *
2655 * @returns VBox status code.
2656 * @param pVM Pointer to the VM.
2657 * @param pVCpu Pointer to the VMCPU.
2658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2659 * out-of-sync. Make sure to update the required fields
2660 * before using them.
2661 *
2662 * @remarks requires EFER.
2663 */
2664DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2665{
2666 int rc = VINF_SUCCESS;
2667 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2668 {
2669 PVM pVM = pVCpu->CTX_SUFF(pVM);
2670 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2671 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2672
2673 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2674 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2675
2676 /*
2677 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
2678 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
2679 */
2680#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2681 if (HMVMX_IS_64BIT_HOST_MODE())
2682 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2683 else
2684 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2685#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2686 if (CPUMIsGuestInLongModeEx(pMixedCtx))
2687 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2688 else
2689 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2690#endif
2691
2692 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2693 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2694
2695 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2696 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2697 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2698 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2699 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2700
2701 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2702 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2703
2704 if ((val & zap) != val)
2705 {
2706 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2707 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
2708 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
2709 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2710 }
2711
2712 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2713 AssertRCReturn(rc, rc);
2714
2715 /* Update VCPU with the currently set VM-exit controls. */
2716 pVCpu->hm.s.vmx.u32ExitCtls = val;
2717 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2718 }
2719 return rc;
2720}
2721
2722
2723/**
2724 * Loads the guest APIC and related state.
2725 *
2726 * @returns VBox status code.
2727 * @param pVM Pointer to the VM.
2728 * @param pVCpu Pointer to the VMCPU.
2729 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2730 * out-of-sync. Make sure to update the required fields
2731 * before using them.
2732 */
2733DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2734{
2735 int rc = VINF_SUCCESS;
2736 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2737 {
2738 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2739 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2740 {
2741 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2742
2743 bool fPendingIntr = false;
2744 uint8_t u8Tpr = 0;
2745 uint8_t u8PendingIntr = 0;
2746 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2747 AssertRCReturn(rc, rc);
2748
2749 /*
2750 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2751 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2752 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2753 * the interrupt when we VM-exit for other reasons.
2754 */
2755 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2756 uint32_t u32TprThreshold = 0;
2757 if (fPendingIntr)
2758 {
2759 /* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2760 const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2761 const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2762 if (u8PendingPriority <= u8TprPriority)
2763 u32TprThreshold = u8PendingPriority;
2764 else
2765 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2766 }
2767 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2768
2769 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2770 AssertRCReturn(rc, rc);
2771 }
2772
2773 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2774 }
2775 return rc;
2776}
2777
2778
2779/**
2780 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2781 *
2782 * @returns Guest's interruptibility-state.
2783 * @param pVCpu Pointer to the VMCPU.
2784 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2785 * out-of-sync. Make sure to update the required fields
2786 * before using them.
2787 *
2788 * @remarks No-long-jump zone!!!
2789 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2790 */
2791DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2792{
2793 /*
2794 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2795 * inhibit interrupts or clear any existing interrupt-inhibition.
2796 */
2797 uint32_t uIntrState = 0;
2798 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2799 {
2800 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2801 AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS))
2802 == (HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2803 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2804 {
2805 /*
2806 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2807 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
2808 */
2809 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2810 }
2811 else if (pMixedCtx->eflags.Bits.u1IF)
2812 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2813 else
2814 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2815 }
2816 return uIntrState;
2817}
2818
2819
2820/**
2821 * Loads the guest's interruptibility-state into the guest-state area in the
2822 * VMCS.
2823 *
2824 * @returns VBox status code.
2825 * @param pVCpu Pointer to the VMCPU.
2826 * @param uIntrState The interruptibility-state to set.
2827 */
2828static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2829{
2830 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2831 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2832 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2833 AssertRCReturn(rc, rc);
2834 return rc;
2835}
2836
2837
2838/**
2839 * Loads the guest's RIP into the guest-state area in the VMCS.
2840 *
2841 * @returns VBox status code.
2842 * @param pVCpu Pointer to the VMCPU.
2843 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2844 * out-of-sync. Make sure to update the required fields
2845 * before using them.
2846 *
2847 * @remarks No-long-jump zone!!!
2848 */
2849static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2850{
2851 int rc = VINF_SUCCESS;
2852 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2853 {
2854 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2855 AssertRCReturn(rc, rc);
2856 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2857 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#x\n", pMixedCtx->rip, pVCpu->hm.s.fContextUseFlags));
2858 }
2859 return rc;
2860}
2861
2862
2863/**
2864 * Loads the guest's RSP into the guest-state area in the VMCS.
2865 *
2866 * @returns VBox status code.
2867 * @param pVCpu Pointer to the VMCPU.
2868 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2869 * out-of-sync. Make sure to update the required fields
2870 * before using them.
2871 *
2872 * @remarks No-long-jump zone!!!
2873 */
2874static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2875{
2876 int rc = VINF_SUCCESS;
2877 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2878 {
2879 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2880 AssertRCReturn(rc, rc);
2881 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2882 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
2883 }
2884 return rc;
2885}
2886
2887
2888/**
2889 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
2890 *
2891 * @returns VBox status code.
2892 * @param pVCpu Pointer to the VMCPU.
2893 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2894 * out-of-sync. Make sure to update the required fields
2895 * before using them.
2896 *
2897 * @remarks No-long-jump zone!!!
2898 */
2899static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2900{
2901 int rc = VINF_SUCCESS;
2902 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2903 {
2904 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2905 Let us assert it as such and use 32-bit VMWRITE. */
2906 Assert(!(pMixedCtx->rflags.u64 >> 32));
2907 X86EFLAGS Eflags = pMixedCtx->eflags;
2908 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2909 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2910
2911 /*
2912 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2913 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2914 */
2915 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2916 {
2917 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2918 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2919 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
2920 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2921 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2922 }
2923
2924 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
2925 AssertRCReturn(rc, rc);
2926
2927 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2928 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
2929 }
2930 return rc;
2931}
2932
2933
2934/**
2935 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2936 *
2937 * @returns VBox status code.
2938 * @param pVCpu Pointer to the VMCPU.
2939 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2940 * out-of-sync. Make sure to update the required fields
2941 * before using them.
2942 *
2943 * @remarks No-long-jump zone!!!
2944 */
2945DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2946{
2947 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2948 AssertRCReturn(rc, rc);
2949 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2950 AssertRCReturn(rc, rc);
2951 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2952 AssertRCReturn(rc, rc);
2953 return rc;
2954}
2955
2956
2957/**
2958 * Loads the guest CR0 control register into the guest-state area in the VMCS.
2959 * CR0 is partially shared with the host and we have to consider the FPU bits.
2960 *
2961 * @returns VBox status code.
2962 * @param pVM Pointer to the VM.
2963 * @param pVCpu Pointer to the VMCPU.
2964 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
2965 * out-of-sync. Make sure to update the required fields
2966 * before using them.
2967 *
2968 * @remarks No-long-jump zone!!!
2969 */
2970static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2971{
2972 /*
2973 * Guest CR0.
2974 * Guest FPU.
2975 */
2976 int rc = VINF_SUCCESS;
2977 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2978 {
2979 Assert(!(pMixedCtx->cr0 >> 32));
2980 uint32_t u32GuestCR0 = pMixedCtx->cr0;
2981 PVM pVM = pVCpu->CTX_SUFF(pVM);
2982
2983 /* The guest's view (read access) of its CR0 is unblemished. */
2984 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2985 AssertRCReturn(rc, rc);
2986 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2987
2988 /* Setup VT-x's view of the guest CR0. */
2989 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2990 if (pVM->hm.s.fNestedPaging)
2991 {
2992 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
2993 {
2994 /* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2995 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2996 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2997 }
2998 else
2999 {
3000 /* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
3001 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3002 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3003 }
3004
3005 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3006 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3007 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3008
3009 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3010 AssertRCReturn(rc, rc);
3011 }
3012 else
3013 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3014
3015 /*
3016 * Guest FPU bits.
3017 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3018 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3019 */
3020 u32GuestCR0 |= X86_CR0_NE;
3021 bool fInterceptNM = false;
3022 if (CPUMIsGuestFPUStateActive(pVCpu))
3023 {
3024 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3025 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3026 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3027 }
3028 else
3029 {
3030 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3031 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3032 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3033 }
3034
3035 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3036 bool fInterceptMF = false;
3037 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3038 fInterceptMF = true;
3039
3040 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3041 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3042 {
3043 Assert(PDMVmmDevHeapIsEnabled(pVM));
3044 Assert(pVM->hm.s.vmx.pRealModeTSS);
3045 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3046 fInterceptNM = true;
3047 fInterceptMF = true;
3048 }
3049 else
3050 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3051
3052 if (fInterceptNM)
3053 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3054 else
3055 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3056
3057 if (fInterceptMF)
3058 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3059 else
3060 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3061
3062 /* Additional intercepts for debugging, define these yourself explicitly. */
3063#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3064 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3065 | RT_BIT(X86_XCPT_BP)
3066 | RT_BIT(X86_XCPT_DB)
3067 | RT_BIT(X86_XCPT_DE)
3068 | RT_BIT(X86_XCPT_NM)
3069 | RT_BIT(X86_XCPT_UD)
3070 | RT_BIT(X86_XCPT_NP)
3071 | RT_BIT(X86_XCPT_SS)
3072 | RT_BIT(X86_XCPT_GP)
3073 | RT_BIT(X86_XCPT_PF)
3074 | RT_BIT(X86_XCPT_MF)
3075 ;
3076#elif defined(HMVMX_ALWAYS_TRAP_PF)
3077 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3078#endif
3079
3080 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3081
3082 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3083 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3084 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3085 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3086 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3087 else
3088 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3089
3090 u32GuestCR0 |= uSetCR0;
3091 u32GuestCR0 &= uZapCR0;
3092 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3093
3094 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3095 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3096 AssertRCReturn(rc, rc);
3097 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3098 AssertRCReturn(rc, rc);
3099 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3100
3101 /*
3102 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3103 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3104 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3105 */
3106 uint32_t u32CR0Mask = 0;
3107 u32CR0Mask = X86_CR0_PE
3108 | X86_CR0_NE
3109 | X86_CR0_WP
3110 | X86_CR0_PG
3111 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3112 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3113 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3114 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3115 u32CR0Mask &= ~X86_CR0_PE;
3116 if (pVM->hm.s.fNestedPaging)
3117 u32CR0Mask &= ~X86_CR0_WP;
3118
3119 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3120 if (fInterceptNM)
3121 u32CR0Mask |= (X86_CR0_TS | X86_CR0_MP);
3122 else
3123 u32CR0Mask &= ~(X86_CR0_TS | X86_CR0_MP);
3124
3125 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3126 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3127 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3128 AssertRCReturn(rc, rc);
3129
3130 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
3131 }
3132 return rc;
3133}
3134
3135
3136/**
3137 * Loads the guest control registers (CR3, CR4) into the guest-state area
3138 * in the VMCS.
3139 *
3140 * @returns VBox status code.
3141 * @param pVM Pointer to the VM.
3142 * @param pVCpu Pointer to the VMCPU.
3143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3144 * out-of-sync. Make sure to update the required fields
3145 * before using them.
3146 *
3147 * @remarks No-long-jump zone!!!
3148 */
3149static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3150{
3151 int rc = VINF_SUCCESS;
3152 PVM pVM = pVCpu->CTX_SUFF(pVM);
3153
3154 /*
3155 * Guest CR2.
3156 * It's always loaded in the assembler code. Nothing to do here.
3157 */
3158
3159 /*
3160 * Guest CR3.
3161 */
3162 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
3163 {
3164 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3165 if (pVM->hm.s.fNestedPaging)
3166 {
3167 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3168
3169 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3170 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3171 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3172 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3173
3174 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3175 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3176 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3177
3178 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3179 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3180 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3181 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3182
3183 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3184 AssertRCReturn(rc, rc);
3185 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3186
3187 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3188 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3189 {
3190 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3191 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3192 {
3193 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3194 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3195 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3196 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3197 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3198 }
3199
3200 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3201 have Unrestricted Execution to handle the guest when it's not using paging. */
3202 GCPhysGuestCR3 = pMixedCtx->cr3;
3203 }
3204 else
3205 {
3206 /*
3207 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3208 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3209 * EPT takes care of translating it to host-physical addresses.
3210 */
3211 RTGCPHYS GCPhys;
3212 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3213 Assert(PDMVmmDevHeapIsEnabled(pVM));
3214
3215 /* We obtain it here every time as the guest could have relocated this PCI region. */
3216 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3217 AssertRCReturn(rc, rc);
3218
3219 GCPhysGuestCR3 = GCPhys;
3220 }
3221
3222 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3223 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3224 }
3225 else
3226 {
3227 /* Non-nested paging case, just use the hypervisor's CR3. */
3228 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3229
3230 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3231 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3232 }
3233 AssertRCReturn(rc, rc);
3234
3235 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
3236 }
3237
3238 /*
3239 * Guest CR4.
3240 */
3241 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
3242 {
3243 Assert(!(pMixedCtx->cr4 >> 32));
3244 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3245
3246 /* The guest's view of its CR4 is unblemished. */
3247 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3248 AssertRCReturn(rc, rc);
3249 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3250
3251 /* Setup VT-x's view of the guest CR4. */
3252 /*
3253 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3254 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3255 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3256 */
3257 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3258 {
3259 Assert(pVM->hm.s.vmx.pRealModeTSS);
3260 Assert(PDMVmmDevHeapIsEnabled(pVM));
3261 u32GuestCR4 &= ~X86_CR4_VME;
3262 }
3263
3264 if (pVM->hm.s.fNestedPaging)
3265 {
3266 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3267 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3268 {
3269 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3270 u32GuestCR4 |= X86_CR4_PSE;
3271 /* Our identity mapping is a 32 bits page directory. */
3272 u32GuestCR4 &= ~X86_CR4_PAE;
3273 }
3274 /* else use guest CR4.*/
3275 }
3276 else
3277 {
3278 /*
3279 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3280 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3281 */
3282 switch (pVCpu->hm.s.enmShadowMode)
3283 {
3284 case PGMMODE_REAL: /* Real-mode. */
3285 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3286 case PGMMODE_32_BIT: /* 32-bit paging. */
3287 {
3288 u32GuestCR4 &= ~X86_CR4_PAE;
3289 break;
3290 }
3291
3292 case PGMMODE_PAE: /* PAE paging. */
3293 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3294 {
3295 u32GuestCR4 |= X86_CR4_PAE;
3296 break;
3297 }
3298
3299 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3300 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3301#ifdef VBOX_ENABLE_64_BITS_GUESTS
3302 break;
3303#endif
3304 default:
3305 AssertFailed();
3306 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3307 }
3308 }
3309
3310 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3311 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3312 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3313 u32GuestCR4 |= uSetCR4;
3314 u32GuestCR4 &= uZapCR4;
3315
3316 /* Write VT-x's view of the guest CR4 into the VMCS. */
3317 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3318 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3319 AssertRCReturn(rc, rc);
3320
3321 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3322 uint32_t u32CR4Mask = 0;
3323 u32CR4Mask = X86_CR4_VME
3324 | X86_CR4_PAE
3325 | X86_CR4_PGE
3326 | X86_CR4_PSE
3327 | X86_CR4_VMXE;
3328 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3329 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3330 AssertRCReturn(rc, rc);
3331
3332 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3333 }
3334 return rc;
3335}
3336
3337
3338/**
3339 * Loads the guest debug registers into the guest-state area in the VMCS.
3340 * This also sets up whether #DB and MOV DRx accesses cause VM exits.
3341 *
3342 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
3343 *
3344 * @returns VBox status code.
3345 * @param pVCpu Pointer to the VMCPU.
3346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3347 * out-of-sync. Make sure to update the required fields
3348 * before using them.
3349 *
3350 * @remarks No-long-jump zone!!!
3351 */
3352static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3353{
3354 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3355 return VINF_SUCCESS;
3356
3357#ifdef VBOX_STRICT
3358 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3359 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3360 {
3361 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3362 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
3363 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
3364 }
3365#endif
3366
3367 int rc;
3368 PVM pVM = pVCpu->CTX_SUFF(pVM);
3369 bool fInterceptDB = false;
3370 bool fInterceptMovDRx = false;
3371 if (pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu))
3372 {
3373 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3374 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3375 {
3376 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3377 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3378 AssertRCReturn(rc, rc);
3379 Assert(fInterceptDB == false);
3380 }
3381 else
3382 {
3383 pMixedCtx->eflags.u32 |= X86_EFL_TF;
3384 pVCpu->hm.s.fClearTrapFlag = true;
3385 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RFLAGS;
3386 fInterceptDB = true;
3387 }
3388 }
3389
3390 if (fInterceptDB || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
3391 {
3392 /*
3393 * Use the combined guest and host DRx values found in the hypervisor
3394 * register set because the debugger has breakpoints active or someone
3395 * is single stepping on the host side without a monitor trap flag.
3396 *
3397 * Note! DBGF expects a clean DR6 state before executing guest code.
3398 */
3399 if (!CPUMIsHyperDebugStateActive(pVCpu))
3400 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
3401 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
3402 Assert(CPUMIsHyperDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3403
3404 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
3405 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
3406 AssertRCReturn(rc, rc);
3407
3408 fInterceptDB = true;
3409 fInterceptMovDRx = true;
3410 }
3411 else
3412 {
3413 /*
3414 * If the guest has enabled debug registers, we need to load them prior to
3415 * executing guest code so they'll trigger at the right time.
3416 */
3417 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
3418 {
3419 if (!CPUMIsGuestDebugStateActive(pVCpu))
3420 {
3421 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
3422 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3423 }
3424 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
3425 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
3426 }
3427 /*
3428 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
3429 * must intercept #DB in order to maintain a correct DR6 guest value.
3430 */
3431 else if (!CPUMIsGuestDebugStateActive(pVCpu))
3432 {
3433 fInterceptMovDRx = true;
3434 fInterceptDB = true;
3435 }
3436
3437 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3438 AssertRCReturn(rc, rc);
3439 }
3440
3441 /*
3442 * Update the exception bitmap regarding intercepting #DB generated by the guest.
3443 */
3444 if (fInterceptDB)
3445 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
3446 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3447 {
3448#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3449 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3450#endif
3451 }
3452 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3453 AssertRCReturn(rc, rc);
3454
3455 /*
3456 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
3457 */
3458 if (fInterceptMovDRx)
3459 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3460 else
3461 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3462 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3463 AssertRCReturn(rc, rc);
3464
3465 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3466 return VINF_SUCCESS;
3467}
3468
3469
3470#ifdef VBOX_STRICT
3471/**
3472 * Strict function to validate segment registers.
3473 *
3474 * @remarks ASSUMES CR0 is up to date.
3475 */
3476static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3477{
3478 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3479 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3480 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
3481 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3482 && ( !CPUMIsGuestInRealModeEx(pCtx)
3483 && !CPUMIsGuestInV86ModeEx(pCtx)))
3484 {
3485 /* Protected mode checks */
3486 /* CS */
3487 Assert(pCtx->cs.Attr.n.u1Present);
3488 Assert(!(pCtx->cs.Attr.u & 0xf00));
3489 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3490 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3491 || !(pCtx->cs.Attr.n.u1Granularity));
3492 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3493 || (pCtx->cs.Attr.n.u1Granularity));
3494 /* CS cannot be loaded with NULL in protected mode. */
3495 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3496 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
3497 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3498 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
3499 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3500 else
3501 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3502 /* SS */
3503 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3504 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3505 if ( !(pCtx->cr0 & X86_CR0_PE)
3506 || pCtx->cs.Attr.n.u4Type == 3)
3507 {
3508 Assert(!pCtx->ss.Attr.n.u2Dpl);
3509 }
3510 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
3511 {
3512 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3513 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
3514 Assert(pCtx->ss.Attr.n.u1Present);
3515 Assert(!(pCtx->ss.Attr.u & 0xf00));
3516 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3517 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3518 || !(pCtx->ss.Attr.n.u1Granularity));
3519 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3520 || (pCtx->ss.Attr.n.u1Granularity));
3521 }
3522 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3523 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
3524 {
3525 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3526 Assert(pCtx->ds.Attr.n.u1Present);
3527 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3528 Assert(!(pCtx->ds.Attr.u & 0xf00));
3529 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3530 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3531 || !(pCtx->ds.Attr.n.u1Granularity));
3532 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3533 || (pCtx->ds.Attr.n.u1Granularity));
3534 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3535 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3536 }
3537 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
3538 {
3539 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3540 Assert(pCtx->es.Attr.n.u1Present);
3541 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3542 Assert(!(pCtx->es.Attr.u & 0xf00));
3543 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3544 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3545 || !(pCtx->es.Attr.n.u1Granularity));
3546 Assert( !(pCtx->es.u32Limit & 0xfff00000)
3547 || (pCtx->es.Attr.n.u1Granularity));
3548 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3549 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3550 }
3551 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
3552 {
3553 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3554 Assert(pCtx->fs.Attr.n.u1Present);
3555 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3556 Assert(!(pCtx->fs.Attr.u & 0xf00));
3557 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3558 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3559 || !(pCtx->fs.Attr.n.u1Granularity));
3560 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3561 || (pCtx->fs.Attr.n.u1Granularity));
3562 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3563 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3564 }
3565 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
3566 {
3567 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3568 Assert(pCtx->gs.Attr.n.u1Present);
3569 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3570 Assert(!(pCtx->gs.Attr.u & 0xf00));
3571 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3572 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3573 || !(pCtx->gs.Attr.n.u1Granularity));
3574 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3575 || (pCtx->gs.Attr.n.u1Granularity));
3576 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3577 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3578 }
3579 /* 64-bit capable CPUs. */
3580# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3581 Assert(!(pCtx->cs.u64Base >> 32));
3582 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
3583 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
3584 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
3585# endif
3586 }
3587 else if ( CPUMIsGuestInV86ModeEx(pCtx)
3588 || ( CPUMIsGuestInRealModeEx(pCtx)
3589 && !pVM->hm.s.vmx.fUnrestrictedGuest))
3590 {
3591 /* Real and v86 mode checks. */
3592 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3593 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3595 {
3596 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3597 }
3598 else
3599 {
3600 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3601 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3602 }
3603
3604 /* CS */
3605 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3606 Assert(pCtx->cs.u32Limit == 0xffff);
3607 Assert(u32CSAttr == 0xf3);
3608 /* SS */
3609 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3610 Assert(pCtx->ss.u32Limit == 0xffff);
3611 Assert(u32SSAttr == 0xf3);
3612 /* DS */
3613 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3614 Assert(pCtx->ds.u32Limit == 0xffff);
3615 Assert(u32DSAttr == 0xf3);
3616 /* ES */
3617 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3618 Assert(pCtx->es.u32Limit == 0xffff);
3619 Assert(u32ESAttr == 0xf3);
3620 /* FS */
3621 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3622 Assert(pCtx->fs.u32Limit == 0xffff);
3623 Assert(u32FSAttr == 0xf3);
3624 /* GS */
3625 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3626 Assert(pCtx->gs.u32Limit == 0xffff);
3627 Assert(u32GSAttr == 0xf3);
3628 /* 64-bit capable CPUs. */
3629# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3630 Assert(!(pCtx->cs.u64Base >> 32));
3631 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
3632 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
3633 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
3634# endif
3635 }
3636}
3637#endif /* VBOX_STRICT */
3638
3639
3640/**
3641 * Writes a guest segment register into the guest-state area in the VMCS.
3642 *
3643 * @returns VBox status code.
3644 * @param pVCpu Pointer to the VMCPU.
3645 * @param idxSel Index of the selector in the VMCS.
3646 * @param idxLimit Index of the segment limit in the VMCS.
3647 * @param idxBase Index of the segment base in the VMCS.
3648 * @param idxAccess Index of the access rights of the segment in the VMCS.
3649 * @param pSelReg Pointer to the segment selector.
3650 * @param pCtx Pointer to the guest-CPU context.
3651 *
3652 * @remarks No-long-jump zone!!!
3653 */
3654static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3655 uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3656{
3657 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3658 AssertRCReturn(rc, rc);
3659 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3660 AssertRCReturn(rc, rc);
3661 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3662 AssertRCReturn(rc, rc);
3663
3664 uint32_t u32Access = pSelReg->Attr.u;
3665 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3666 {
3667 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3668 u32Access = 0xf3;
3669 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3670 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3671 }
3672 else
3673 {
3674 /*
3675 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3676 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3677 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3678 * loaded in protected-mode have their attribute as 0.
3679 */
3680 if (!u32Access)
3681 u32Access = X86DESCATTR_UNUSABLE;
3682 }
3683
3684 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3685 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
3686 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3687
3688 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3689 AssertRCReturn(rc, rc);
3690 return rc;
3691}
3692
3693
3694/**
3695 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3696 * into the guest-state area in the VMCS.
3697 *
3698 * @returns VBox status code.
3699 * @param pVM Pointer to the VM.
3700 * @param pVCPU Pointer to the VMCPU.
3701 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3702 * out-of-sync. Make sure to update the required fields
3703 * before using them.
3704 *
3705 * @remarks ASSUMES CR0 is up to date (strict builds validation).
3706 * @remarks No-long-jump zone!!!
3707 */
3708static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3709{
3710 int rc = VERR_INTERNAL_ERROR_5;
3711 PVM pVM = pVCpu->CTX_SUFF(pVM);
3712
3713 /*
3714 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
3715 */
3716 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3717 {
3718 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3719 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3720 {
3721 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
3722 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
3723 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
3724 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
3725 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
3726 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
3727 }
3728
3729#ifdef VBOX_WITH_REM
3730 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3731 {
3732 Assert(pVM->hm.s.vmx.pRealModeTSS);
3733 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3734 if ( pVCpu->hm.s.vmx.fWasInRealMode
3735 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3736 {
3737 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3738 in real-mode (e.g. OpenBSD 4.0) */
3739 REMFlushTBs(pVM);
3740 Log4(("Load: Switch to protected mode detected!\n"));
3741 pVCpu->hm.s.vmx.fWasInRealMode = false;
3742 }
3743 }
3744#endif
3745 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3746 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3747 AssertRCReturn(rc, rc);
3748 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3749 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3750 AssertRCReturn(rc, rc);
3751 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3752 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3753 AssertRCReturn(rc, rc);
3754 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3755 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3756 AssertRCReturn(rc, rc);
3757 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3758 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3759 AssertRCReturn(rc, rc);
3760 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3761 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3762 AssertRCReturn(rc, rc);
3763
3764 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3765 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3766#ifdef VBOX_STRICT
3767 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3768#endif
3769 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3770 }
3771
3772 /*
3773 * Guest TR.
3774 */
3775 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3776 {
3777 /*
3778 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3779 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3780 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3781 */
3782 uint16_t u16Sel = 0;
3783 uint32_t u32Limit = 0;
3784 uint64_t u64Base = 0;
3785 uint32_t u32AccessRights = 0;
3786
3787 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3788 {
3789 u16Sel = pMixedCtx->tr.Sel;
3790 u32Limit = pMixedCtx->tr.u32Limit;
3791 u64Base = pMixedCtx->tr.u64Base;
3792 u32AccessRights = pMixedCtx->tr.Attr.u;
3793 }
3794 else
3795 {
3796 Assert(pVM->hm.s.vmx.pRealModeTSS);
3797 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3798
3799 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3800 RTGCPHYS GCPhys;
3801 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3802 AssertRCReturn(rc, rc);
3803
3804 X86DESCATTR DescAttr;
3805 DescAttr.u = 0;
3806 DescAttr.n.u1Present = 1;
3807 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3808
3809 u16Sel = 0;
3810 u32Limit = HM_VTX_TSS_SIZE;
3811 u64Base = GCPhys; /* in real-mode phys = virt. */
3812 u32AccessRights = DescAttr.u;
3813 }
3814
3815 /* Validate. */
3816 Assert(!(u16Sel & RT_BIT(2)));
3817 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3818 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3819 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3820 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3821 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3822 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3823 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3824 Assert( (u32Limit & 0xfff) == 0xfff
3825 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3826 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3827 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3828
3829 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3830 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3831 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3832 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3833
3834 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3835 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3836 }
3837
3838 /*
3839 * Guest GDTR.
3840 */
3841 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3842 {
3843 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3844 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3845
3846 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3847 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3848 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3849 }
3850
3851 /*
3852 * Guest LDTR.
3853 */
3854 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3855 {
3856 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3857 uint32_t u32Access = 0;
3858 if (!pMixedCtx->ldtr.Attr.u)
3859 u32Access = X86DESCATTR_UNUSABLE;
3860 else
3861 u32Access = pMixedCtx->ldtr.Attr.u;
3862
3863 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3864 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3865 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3866 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3867
3868 /* Validate. */
3869 if (!(u32Access & X86DESCATTR_UNUSABLE))
3870 {
3871 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3872 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3873 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3874 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3875 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3876 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3877 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3878 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3879 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3880 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3881 }
3882
3883 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3884 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3885 }
3886
3887 /*
3888 * Guest IDTR.
3889 */
3890 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3891 {
3892 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3893 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3894
3895 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
3896 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3897 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3898 }
3899
3900 return VINF_SUCCESS;
3901}
3902
3903
3904/**
3905 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3906 * areas. These MSRs will automatically be loaded to the host CPU on every
3907 * successful VM entry and stored from the host CPU on every successful VM exit.
3908 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
3909 *
3910 * @returns VBox status code.
3911 * @param pVCpu Pointer to the VMCPU.
3912 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3913 * out-of-sync. Make sure to update the required fields
3914 * before using them.
3915 *
3916 * @remarks No-long-jump zone!!!
3917 */
3918static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3919{
3920 AssertPtr(pVCpu);
3921 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3922
3923 /*
3924 * MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3925 */
3926 int rc = VINF_SUCCESS;
3927 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3928 {
3929#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3930 PVM pVM = pVCpu->CTX_SUFF(pVM);
3931 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3932 uint32_t cGuestMsrs = 0;
3933
3934 /* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3935 /** @todo r=ramshankar: Optimize this further to do lazy restoration and only
3936 * when the guest really is in 64-bit mode. */
3937 bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3938 if (fSupportsLongMode)
3939 {
3940 pGuestMsr->u32Msr = MSR_K8_LSTAR;
3941 pGuestMsr->u32Reserved = 0;
3942 pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3943 pGuestMsr++; cGuestMsrs++;
3944 pGuestMsr->u32Msr = MSR_K6_STAR;
3945 pGuestMsr->u32Reserved = 0;
3946 pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3947 pGuestMsr++; cGuestMsrs++;
3948 pGuestMsr->u32Msr = MSR_K8_SF_MASK;
3949 pGuestMsr->u32Reserved = 0;
3950 pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3951 pGuestMsr++; cGuestMsrs++;
3952 pGuestMsr->u32Msr = MSR_K8_KERNEL_GS_BASE;
3953 pGuestMsr->u32Reserved = 0;
3954 pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3955 pGuestMsr++; cGuestMsrs++;
3956 }
3957
3958 /*
3959 * RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3960 * load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3961 */
3962 if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3963 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3964 {
3965 pGuestMsr->u32Msr = MSR_K8_TSC_AUX;
3966 pGuestMsr->u32Reserved = 0;
3967 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3968 AssertRCReturn(rc, rc);
3969 pGuestMsr++; cGuestMsrs++;
3970 }
3971
3972 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3973 if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.Msrs.u64Misc))
3974 {
3975 LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3976 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
3977 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3978 }
3979
3980 /* Update the VCPU's copy of the guest MSR count. */
3981 pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3982 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3983 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3984#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3985
3986 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3987 }
3988
3989 /*
3990 * Guest Sysenter MSRs.
3991 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3992 * VM-exits on WRMSRs for these MSRs.
3993 */
3994 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3995 {
3996 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3997 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3998 }
3999 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4000 {
4001 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4002 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
4003 }
4004 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4005 {
4006 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4007 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
4008 }
4009
4010 return rc;
4011}
4012
4013
4014/**
4015 * Loads the guest activity state into the guest-state area in the VMCS.
4016 *
4017 * @returns VBox status code.
4018 * @param pVCpu Pointer to the VMCPU.
4019 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4020 * out-of-sync. Make sure to update the required fields
4021 * before using them.
4022 *
4023 * @remarks No-long-jump zone!!!
4024 */
4025static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4026{
4027 /** @todo See if we can make use of other states, e.g.
4028 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4029 int rc = VINF_SUCCESS;
4030 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
4031 {
4032 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4033 AssertRCReturn(rc, rc);
4034 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
4035 }
4036 return rc;
4037}
4038
4039
4040/**
4041 * Sets up the appropriate function to run guest code.
4042 *
4043 * @returns VBox status code.
4044 * @param pVCpu Pointer to the VMCPU.
4045 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4046 * out-of-sync. Make sure to update the required fields
4047 * before using them.
4048 *
4049 * @remarks No-long-jump zone!!!
4050 */
4051static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4052{
4053 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4054 {
4055#ifndef VBOX_ENABLE_64_BITS_GUESTS
4056 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4057#endif
4058 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4059#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4060 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4061 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4062#else
4063 /* 64-bit host or hybrid host. */
4064 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4065#endif
4066 }
4067 else
4068 {
4069 /* Guest is not in long mode, use the 32-bit handler. */
4070#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4071 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4072 {
4073 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4074 /** @todo r=bird: Don't we need to set up the host resume (after
4075 * vmlaunch/vmresume) state here?? I'm forcing a trip to ring-3 now
4076 * in the hope that it will prevent crashing the host. A better
4077 * fix should be found as the guest may be going back and forth
4078 * between 16/32-bit and long mode frequently at times. */
4079 VMCPU_FF_SET(pVCpu, VMCPU_FF_TO_R3);
4080 }
4081#else
4082 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4083#endif
4084 }
4085 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4086 return VINF_SUCCESS;
4087}
4088
4089
4090/**
4091 * Wrapper for running the guest code in VT-x.
4092 *
4093 * @returns VBox strict status code.
4094 * @param pVM Pointer to the VM.
4095 * @param pVCpu Pointer to the VMCPU.
4096 * @param pCtx Pointer to the guest-CPU context.
4097 *
4098 * @remarks No-long-jump zone!!!
4099 */
4100DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4101{
4102 /*
4103 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4104 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4105 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4106 */
4107 const bool fResumeVM = !!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4108 /** @todo Add stats for resume vs launch. */
4109#ifdef VBOX_WITH_KERNEL_USING_XMM
4110 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4111#else
4112 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4113#endif
4114}
4115
4116
4117/**
4118 * Reports world-switch error and dumps some useful debug info.
4119 *
4120 * @param pVM Pointer to the VM.
4121 * @param pVCpu Pointer to the VMCPU.
4122 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4123 * @param pCtx Pointer to the guest-CPU context.
4124 * @param pVmxTransient Pointer to the VMX transient structure (only
4125 * exitReason updated).
4126 */
4127static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4128{
4129 Assert(pVM);
4130 Assert(pVCpu);
4131 Assert(pCtx);
4132 Assert(pVmxTransient);
4133 HMVMX_ASSERT_PREEMPT_SAFE();
4134
4135 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4136 switch (rcVMRun)
4137 {
4138 case VERR_VMX_INVALID_VMXON_PTR:
4139 AssertFailed();
4140 break;
4141 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4142 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4143 {
4144 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4145 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4146 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4147 AssertRC(rc);
4148
4149 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4150 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4151 Cannot do it here as we may have been long preempted. */
4152
4153#ifdef VBOX_STRICT
4154 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4155 pVmxTransient->uExitReason));
4156 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4157 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4158 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4159 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4160 else
4161 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4162 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4163 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4164
4165 /* VMX control bits. */
4166 uint32_t u32Val;
4167 uint64_t u64Val;
4168 HMVMXHCUINTREG uHCReg;
4169 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4170 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4171 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4172 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4173 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4174 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4175 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4176 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4177 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4178 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4179 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4180 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4181 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4182 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4183 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4184 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4185 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4186 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4187 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4188 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4189 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4190 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4191 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4192 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4193 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4194 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4195 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4196 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4197 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4198 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4199 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4200 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4201 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4202 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4203 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4204 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4205 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4206 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4207 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4208 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4209 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4210 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4211
4212 /* Guest bits. */
4213 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4214 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4215 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4216 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4217 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4218 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4219 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4220 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4221
4222 /* Host bits. */
4223 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4224 Log4(("Host CR0 %#RHr\n", uHCReg));
4225 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4226 Log4(("Host CR3 %#RHr\n", uHCReg));
4227 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4228 Log4(("Host CR4 %#RHr\n", uHCReg));
4229
4230 RTGDTR HostGdtr;
4231 PCX86DESCHC pDesc;
4232 ASMGetGDTR(&HostGdtr);
4233 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4234 Log4(("Host CS %#08x\n", u32Val));
4235 if (u32Val < HostGdtr.cbGdt)
4236 {
4237 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4238 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4239 }
4240
4241 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4242 Log4(("Host DS %#08x\n", u32Val));
4243 if (u32Val < HostGdtr.cbGdt)
4244 {
4245 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4246 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4247 }
4248
4249 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4250 Log4(("Host ES %#08x\n", u32Val));
4251 if (u32Val < HostGdtr.cbGdt)
4252 {
4253 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4254 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4255 }
4256
4257 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4258 Log4(("Host FS %#08x\n", u32Val));
4259 if (u32Val < HostGdtr.cbGdt)
4260 {
4261 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4262 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4263 }
4264
4265 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4266 Log4(("Host GS %#08x\n", u32Val));
4267 if (u32Val < HostGdtr.cbGdt)
4268 {
4269 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4270 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4271 }
4272
4273 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4274 Log4(("Host SS %#08x\n", u32Val));
4275 if (u32Val < HostGdtr.cbGdt)
4276 {
4277 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4278 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4279 }
4280
4281 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4282 Log4(("Host TR %#08x\n", u32Val));
4283 if (u32Val < HostGdtr.cbGdt)
4284 {
4285 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4286 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
4287 }
4288
4289 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
4290 Log4(("Host TR Base %#RHv\n", uHCReg));
4291 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
4292 Log4(("Host GDTR Base %#RHv\n", uHCReg));
4293 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
4294 Log4(("Host IDTR Base %#RHv\n", uHCReg));
4295 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4296 Log4(("Host SYSENTER CS %#08x\n", u32Val));
4297 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4298 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
4299 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4300 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
4301 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4302 Log4(("Host RSP %#RHv\n", uHCReg));
4303 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4304 Log4(("Host RIP %#RHv\n", uHCReg));
4305# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4306 if (HMVMX_IS_64BIT_HOST_MODE())
4307 {
4308 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4309 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4310 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4311 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4312 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4313 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4314 }
4315# endif
4316#endif /* VBOX_STRICT */
4317 break;
4318 }
4319
4320 default:
4321 /* Impossible */
4322 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4323 break;
4324 }
4325 NOREF(pVM);
4326}
4327
4328
4329#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4330#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4331# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4332#endif
4333#ifdef VBOX_STRICT
4334static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4335{
4336 switch (idxField)
4337 {
4338 case VMX_VMCS_GUEST_RIP:
4339 case VMX_VMCS_GUEST_RSP:
4340 case VMX_VMCS_GUEST_SYSENTER_EIP:
4341 case VMX_VMCS_GUEST_SYSENTER_ESP:
4342 case VMX_VMCS_GUEST_GDTR_BASE:
4343 case VMX_VMCS_GUEST_IDTR_BASE:
4344 case VMX_VMCS_GUEST_CS_BASE:
4345 case VMX_VMCS_GUEST_DS_BASE:
4346 case VMX_VMCS_GUEST_ES_BASE:
4347 case VMX_VMCS_GUEST_FS_BASE:
4348 case VMX_VMCS_GUEST_GS_BASE:
4349 case VMX_VMCS_GUEST_SS_BASE:
4350 case VMX_VMCS_GUEST_LDTR_BASE:
4351 case VMX_VMCS_GUEST_TR_BASE:
4352 case VMX_VMCS_GUEST_CR3:
4353 return true;
4354 }
4355 return false;
4356}
4357
4358static bool hmR0VmxIsValidReadField(uint32_t idxField)
4359{
4360 switch (idxField)
4361 {
4362 /* Read-only fields. */
4363 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4364 return true;
4365 }
4366 /* Remaining readable fields should also be writable. */
4367 return hmR0VmxIsValidWriteField(idxField);
4368}
4369#endif /* VBOX_STRICT */
4370
4371
4372/**
4373 * Executes the specified handler in 64-bit mode.
4374 *
4375 * @returns VBox status code.
4376 * @param pVM Pointer to the VM.
4377 * @param pVCpu Pointer to the VMCPU.
4378 * @param pCtx Pointer to the guest CPU context.
4379 * @param enmOp The operation to perform.
4380 * @param cbParam Number of parameters.
4381 * @param paParam Array of 32-bit parameters.
4382 */
4383VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4384 uint32_t *paParam)
4385{
4386 int rc, rc2;
4387 PHMGLOBALCPUINFO pCpu;
4388 RTHCPHYS HCPhysCpuPage;
4389 RTCCUINTREG uOldEflags;
4390
4391 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4392 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4393 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4394 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4395
4396#ifdef VBOX_STRICT
4397 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4398 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4399
4400 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4401 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4402#endif
4403
4404 /* Disable interrupts. */
4405 uOldEflags = ASMIntDisableFlags();
4406
4407#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4408 RTCPUID idHostCpu = RTMpCpuId();
4409 CPUMR0SetLApic(pVCpu, idHostCpu);
4410#endif
4411
4412 pCpu = HMR0GetCurrentCpu();
4413 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4414
4415 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4416 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4417
4418 /* Leave VMX Root Mode. */
4419 VMXDisable();
4420
4421 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4422
4423 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4424 CPUMSetHyperEIP(pVCpu, enmOp);
4425 for (int i = (int)cbParam - 1; i >= 0; i--)
4426 CPUMPushHyper(pVCpu, paParam[i]);
4427
4428 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4429
4430 /* Call the switcher. */
4431 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4432 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4433
4434 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4435 /* Make sure the VMX instructions don't cause #UD faults. */
4436 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4437
4438 /* Re-enter VMX Root Mode */
4439 rc2 = VMXEnable(HCPhysCpuPage);
4440 if (RT_FAILURE(rc2))
4441 {
4442 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4443 ASMSetFlags(uOldEflags);
4444 return rc2;
4445 }
4446
4447 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
4448 AssertRC(rc2);
4449 Assert(!(ASMGetFlags() & X86_EFL_IF));
4450 ASMSetFlags(uOldEflags);
4451 return rc;
4452}
4453
4454
4455/**
4456 * Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4457 * supporting 64-bit guests.
4458 *
4459 * @returns VBox status code.
4460 * @param fResume Whether to VMLAUNCH or VMRESUME.
4461 * @param pCtx Pointer to the guest-CPU context.
4462 * @param pCache Pointer to the VMCS cache.
4463 * @param pVM Pointer to the VM.
4464 * @param pVCpu Pointer to the VMCPU.
4465 */
4466DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4467{
4468 uint32_t aParam[6];
4469 PHMGLOBALCPUINFO pCpu = NULL;
4470 RTHCPHYS HCPhysCpuPage = 0;
4471 int rc = VERR_INTERNAL_ERROR_5;
4472
4473 pCpu = HMR0GetCurrentCpu();
4474 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4475
4476#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4477 pCache->uPos = 1;
4478 pCache->interPD = PGMGetInterPaeCR3(pVM);
4479 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4480#endif
4481
4482#ifdef VBOX_STRICT
4483 pCache->TestIn.HCPhysCpuPage = 0;
4484 pCache->TestIn.HCPhysVmcs = 0;
4485 pCache->TestIn.pCache = 0;
4486 pCache->TestOut.HCPhysVmcs = 0;
4487 pCache->TestOut.pCache = 0;
4488 pCache->TestOut.pCtx = 0;
4489 pCache->TestOut.eflags = 0;
4490#endif
4491
4492 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4493 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4494 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4495 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4496 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4497 aParam[5] = 0;
4498
4499#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4500 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4501 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4502#endif
4503 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4504
4505#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4506 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4507 Assert(pCtx->dr[4] == 10);
4508 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4509#endif
4510
4511#ifdef VBOX_STRICT
4512 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4513 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4514 pVCpu->hm.s.vmx.HCPhysVmcs));
4515 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4516 pCache->TestOut.HCPhysVmcs));
4517 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4518 pCache->TestOut.pCache));
4519 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4520 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4521 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4522 pCache->TestOut.pCtx));
4523 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4524#endif
4525 return rc;
4526}
4527
4528
4529/**
4530 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4531 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4532 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4533 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4534 *
4535 * @returns VBox status code.
4536 * @param pVM Pointer to the VM.
4537 * @param pVCpu Pointer to the VMCPU.
4538 */
4539static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4540{
4541#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4542{ \
4543 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4544 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4545 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4546 ++cReadFields; \
4547}
4548
4549 AssertPtr(pVM);
4550 AssertPtr(pVCpu);
4551 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4552 uint32_t cReadFields = 0;
4553
4554 /*
4555 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
4556 * and serve to indicate exceptions to the rules.
4557 */
4558
4559 /* Guest-natural selector base fields. */
4560#if 0
4561 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4562 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4563 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4564#endif
4565 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4566 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4567 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4568 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4569 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4570 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4571 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4572 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4573 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4574 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4575 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4576 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4577#if 0
4578 /* Unused natural width guest-state fields. */
4579 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4580 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4581#endif
4582 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4583 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4584
4585 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4586#if 0
4587 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4588 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4589 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4590 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4591 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4592 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4593 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4594 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4595 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4596#endif
4597
4598 /* Natural width guest-state fields. */
4599 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4600#if 0
4601 /* Currently unused field. */
4602 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4603#endif
4604
4605 if (pVM->hm.s.fNestedPaging)
4606 {
4607 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4608 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4609 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4610 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4611 }
4612 else
4613 {
4614 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4615 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4616 }
4617
4618#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4619 return VINF_SUCCESS;
4620}
4621
4622
4623/**
4624 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4625 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4626 * darwin, running 64-bit guests).
4627 *
4628 * @returns VBox status code.
4629 * @param pVCpu Pointer to the VMCPU.
4630 * @param idxField The VMCS field encoding.
4631 * @param u64Val 16, 32 or 64 bits value.
4632 */
4633VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4634{
4635 int rc;
4636 switch (idxField)
4637 {
4638 /*
4639 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4640 */
4641 /* 64-bit Control fields. */
4642 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4643 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4644 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4645 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4646 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4647 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4648 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4649 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4650 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4651 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4652 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4653 case VMX_VMCS64_CTRL_EPTP_FULL:
4654 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4655 /* 64-bit Guest-state fields. */
4656 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4657 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4658 case VMX_VMCS64_GUEST_PAT_FULL:
4659 case VMX_VMCS64_GUEST_EFER_FULL:
4660 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4661 case VMX_VMCS64_GUEST_PDPTE0_FULL:
4662 case VMX_VMCS64_GUEST_PDPTE1_FULL:
4663 case VMX_VMCS64_GUEST_PDPTE2_FULL:
4664 case VMX_VMCS64_GUEST_PDPTE3_FULL:
4665 /* 64-bit Host-state fields. */
4666 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4667 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4668 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4669 {
4670 rc = VMXWriteVmcs32(idxField, u64Val);
4671 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4672 break;
4673 }
4674
4675 /*
4676 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4677 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4678 */
4679 /* Natural-width Guest-state fields. */
4680 case VMX_VMCS_GUEST_CR3:
4681 case VMX_VMCS_GUEST_ES_BASE:
4682 case VMX_VMCS_GUEST_CS_BASE:
4683 case VMX_VMCS_GUEST_SS_BASE:
4684 case VMX_VMCS_GUEST_DS_BASE:
4685 case VMX_VMCS_GUEST_FS_BASE:
4686 case VMX_VMCS_GUEST_GS_BASE:
4687 case VMX_VMCS_GUEST_LDTR_BASE:
4688 case VMX_VMCS_GUEST_TR_BASE:
4689 case VMX_VMCS_GUEST_GDTR_BASE:
4690 case VMX_VMCS_GUEST_IDTR_BASE:
4691 case VMX_VMCS_GUEST_RSP:
4692 case VMX_VMCS_GUEST_RIP:
4693 case VMX_VMCS_GUEST_SYSENTER_ESP:
4694 case VMX_VMCS_GUEST_SYSENTER_EIP:
4695 {
4696 if (!(u64Val >> 32))
4697 {
4698 /* If this field is 64-bit, VT-x will zero out the top bits. */
4699 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4700 }
4701 else
4702 {
4703 /* Assert that only the 32->64 switcher case should ever come here. */
4704 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4705 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4706 }
4707 break;
4708 }
4709
4710 default:
4711 {
4712 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4713 rc = VERR_INVALID_PARAMETER;
4714 break;
4715 }
4716 }
4717 AssertRCReturn(rc, rc);
4718 return rc;
4719}
4720
4721
4722/**
4723 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4724 * hosts (except darwin) for 64-bit guests.
4725 *
4726 * @param pVCpu Pointer to the VMCPU.
4727 * @param idxField The VMCS field encoding.
4728 * @param u64Val 16, 32 or 64 bits value.
4729 */
4730VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4731{
4732 AssertPtr(pVCpu);
4733 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4734
4735 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4736 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4737
4738 /* Make sure there are no duplicates. */
4739 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4740 {
4741 if (pCache->Write.aField[i] == idxField)
4742 {
4743 pCache->Write.aFieldVal[i] = u64Val;
4744 return VINF_SUCCESS;
4745 }
4746 }
4747
4748 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4749 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4750 pCache->Write.cValidEntries++;
4751 return VINF_SUCCESS;
4752}
4753
4754/* Enable later when the assembly code uses these as callbacks. */
4755#if 0
4756/*
4757 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4758 *
4759 * @param pVCpu Pointer to the VMCPU.
4760 * @param pCache Pointer to the VMCS cache.
4761 *
4762 * @remarks No-long-jump zone!!!
4763 */
4764VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4765{
4766 AssertPtr(pCache);
4767 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4768 {
4769 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4770 AssertRC(rc);
4771 }
4772 pCache->Write.cValidEntries = 0;
4773}
4774
4775
4776/**
4777 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
4778 *
4779 * @param pVCpu Pointer to the VMCPU.
4780 * @param pCache Pointer to the VMCS cache.
4781 *
4782 * @remarks No-long-jump zone!!!
4783 */
4784VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4785{
4786 AssertPtr(pCache);
4787 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4788 {
4789 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4790 AssertRC(rc);
4791 }
4792}
4793#endif
4794#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4795
4796
4797/**
4798 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4799 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4800 * timer.
4801 *
4802 * @returns VBox status code.
4803 * @param pVCpu Pointer to the VMCPU.
4804 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4805 * out-of-sync. Make sure to update the required fields
4806 * before using them.
4807 * @remarks No-long-jump zone!!!
4808 */
4809static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4810{
4811 int rc = VERR_INTERNAL_ERROR_5;
4812 bool fOffsettedTsc = false;
4813 PVM pVM = pVCpu->CTX_SUFF(pVM);
4814 if (pVM->hm.s.vmx.fUsePreemptTimer)
4815 {
4816 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4817
4818 /* Make sure the returned values have sane upper and lower boundaries. */
4819 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4820 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4821 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4822 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4823
4824 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4825 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4826 }
4827 else
4828 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4829
4830 if (fOffsettedTsc)
4831 {
4832 uint64_t u64CurTSC = ASMReadTSC();
4833 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4834 {
4835 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4836 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4837
4838 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4839 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4840 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4841 }
4842 else
4843 {
4844 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4845 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4846 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4847 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4848 }
4849 }
4850 else
4851 {
4852 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4853 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4854 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4855 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4856 }
4857}
4858
4859
4860/**
4861 * Determines if an exception is a contributory exception. Contributory
4862 * exceptions are ones which can cause double-faults. Page-fault is
4863 * intentionally not included here as it's a conditional contributory exception.
4864 *
4865 * @returns true if the exception is contributory, false otherwise.
4866 * @param uVector The exception vector.
4867 */
4868DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4869{
4870 switch (uVector)
4871 {
4872 case X86_XCPT_GP:
4873 case X86_XCPT_SS:
4874 case X86_XCPT_NP:
4875 case X86_XCPT_TS:
4876 case X86_XCPT_DE:
4877 return true;
4878 default:
4879 break;
4880 }
4881 return false;
4882}
4883
4884
4885/**
4886 * Sets an event as a pending event to be injected into the guest.
4887 *
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param u32IntrInfo The VM-entry interruption-information field.
4890 * @param cbInstr The VM-entry instruction length in bytes (for software
4891 * interrupts, exceptions and privileged software
4892 * exceptions).
4893 * @param u32ErrCode The VM-entry exception error code.
4894 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4895 * page-fault.
4896 *
4897 * @remarks Statistics counter assumes this is a guest event being injected or
4898 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
4899 * always incremented.
4900 */
4901DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4902 RTGCUINTPTR GCPtrFaultAddress)
4903{
4904 Assert(!pVCpu->hm.s.Event.fPending);
4905 pVCpu->hm.s.Event.fPending = true;
4906 pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4907 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4908 pVCpu->hm.s.Event.cbInstr = cbInstr;
4909 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4910
4911 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
4912}
4913
4914
4915/**
4916 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4917 *
4918 * @param pVCpu Pointer to the VMCPU.
4919 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4920 * out-of-sync. Make sure to update the required fields
4921 * before using them.
4922 */
4923DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4924{
4925 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
4926 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4927 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4928 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
4929}
4930
4931
4932/**
4933 * Handle a condition that occurred while delivering an event through the guest
4934 * IDT.
4935 *
4936 * @returns VBox status code (informational error codes included).
4937 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
4938 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
4939 * continue execution of the guest which will delivery the #DF.
4940 * @retval VINF_EM_RESET if we detected a triple-fault condition.
4941 *
4942 * @param pVCpu Pointer to the VMCPU.
4943 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4944 * out-of-sync. Make sure to update the required fields
4945 * before using them.
4946 * @param pVmxTransient Pointer to the VMX transient structure.
4947 *
4948 * @remarks No-long-jump zone!!!
4949 */
4950static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4951{
4952 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4953 AssertRC(rc);
4954 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4955 {
4956 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4957 AssertRCReturn(rc, rc);
4958
4959 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4960 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4961 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4962
4963 typedef enum
4964 {
4965 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4966 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4967 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4968 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4969 } VMXREFLECTXCPT;
4970
4971 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4972 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4973 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo))
4974 {
4975 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4976 {
4977 enmReflect = VMXREFLECTXCPT_XCPT;
4978#ifdef VBOX_STRICT
4979 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4980 && uExitVector == X86_XCPT_PF)
4981 {
4982 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4983 }
4984#endif
4985 if ( uExitVector == X86_XCPT_PF
4986 && uIdtVector == X86_XCPT_PF)
4987 {
4988 pVmxTransient->fVectoringPF = true;
4989 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
4990 }
4991 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4992 && hmR0VmxIsContributoryXcpt(uExitVector)
4993 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
4994 || uIdtVector == X86_XCPT_PF))
4995 {
4996 enmReflect = VMXREFLECTXCPT_DF;
4997 }
4998 else if (uIdtVector == X86_XCPT_DF)
4999 enmReflect = VMXREFLECTXCPT_TF;
5000 }
5001 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5002 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5003 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5004 {
5005 /*
5006 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
5007 * (whatever they are) as they reoccur when restarting the instruction.
5008 */
5009 enmReflect = VMXREFLECTXCPT_XCPT;
5010 }
5011 }
5012 else
5013 {
5014 /*
5015 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5016 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5017 * original exception to the guest after handling the VM-exit.
5018 */
5019 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5020 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5021 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5022 {
5023 enmReflect = VMXREFLECTXCPT_XCPT;
5024 }
5025 }
5026
5027 switch (enmReflect)
5028 {
5029 case VMXREFLECTXCPT_XCPT:
5030 {
5031 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5032 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5033 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5034
5035 uint32_t u32ErrCode = 0;
5036 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5037 {
5038 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5039 AssertRCReturn(rc, rc);
5040 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5041 }
5042
5043 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5044 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5045 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5046 rc = VINF_SUCCESS;
5047 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5048 pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
5049
5050 break;
5051 }
5052
5053 case VMXREFLECTXCPT_DF:
5054 {
5055 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5056 rc = VINF_HM_DOUBLE_FAULT;
5057 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5058 pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
5059
5060 break;
5061 }
5062
5063 case VMXREFLECTXCPT_TF:
5064 {
5065 rc = VINF_EM_RESET;
5066 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5067 uExitVector));
5068 break;
5069 }
5070
5071 default:
5072 Assert(rc == VINF_SUCCESS);
5073 break;
5074 }
5075 }
5076 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5077 return rc;
5078}
5079
5080
5081/**
5082 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5083 *
5084 * @returns VBox status code.
5085 * @param pVCpu Pointer to the VMCPU.
5086 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087 * out-of-sync. Make sure to update the required fields
5088 * before using them.
5089 *
5090 * @remarks No-long-jump zone!!!
5091 */
5092static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093{
5094 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
5095 {
5096 uint32_t uVal = 0;
5097 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5098 AssertRCReturn(rc, rc);
5099 uint32_t uShadow = 0;
5100 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5101 AssertRCReturn(rc, rc);
5102
5103 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5104 CPUMSetGuestCR0(pVCpu, uVal);
5105 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR0;
5106 }
5107 return VINF_SUCCESS;
5108}
5109
5110
5111/**
5112 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5113 *
5114 * @returns VBox status code.
5115 * @param pVCpu Pointer to the VMCPU.
5116 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5117 * out-of-sync. Make sure to update the required fields
5118 * before using them.
5119 *
5120 * @remarks No-long-jump zone!!!
5121 */
5122static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5123{
5124 int rc = VINF_SUCCESS;
5125 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
5126 {
5127 uint32_t uVal = 0;
5128 uint32_t uShadow = 0;
5129 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5130 AssertRCReturn(rc, rc);
5131 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5132 AssertRCReturn(rc, rc);
5133
5134 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5135 CPUMSetGuestCR4(pVCpu, uVal);
5136 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR4;
5137 }
5138 return rc;
5139}
5140
5141
5142/**
5143 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5144 *
5145 * @returns VBox status code.
5146 * @param pVCpu Pointer to the VMCPU.
5147 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5148 * out-of-sync. Make sure to update the required fields
5149 * before using them.
5150 *
5151 * @remarks No-long-jump zone!!!
5152 */
5153static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5154{
5155 int rc = VINF_SUCCESS;
5156 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
5157 {
5158 uint64_t u64Val = 0;
5159 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5160 AssertRCReturn(rc, rc);
5161
5162 pMixedCtx->rip = u64Val;
5163 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RIP;
5164 }
5165 return rc;
5166}
5167
5168
5169/**
5170 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5171 *
5172 * @returns VBox status code.
5173 * @param pVCpu Pointer to the VMCPU.
5174 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5175 * out-of-sync. Make sure to update the required fields
5176 * before using them.
5177 *
5178 * @remarks No-long-jump zone!!!
5179 */
5180static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5181{
5182 int rc = VINF_SUCCESS;
5183 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
5184 {
5185 uint64_t u64Val = 0;
5186 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5187 AssertRCReturn(rc, rc);
5188
5189 pMixedCtx->rsp = u64Val;
5190 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RSP;
5191 }
5192 return rc;
5193}
5194
5195
5196/**
5197 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5198 *
5199 * @returns VBox status code.
5200 * @param pVCpu Pointer to the VMCPU.
5201 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5202 * out-of-sync. Make sure to update the required fields
5203 * before using them.
5204 *
5205 * @remarks No-long-jump zone!!!
5206 */
5207static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5208{
5209 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
5210 {
5211 uint32_t uVal = 0;
5212 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5213 AssertRCReturn(rc, rc);
5214
5215 pMixedCtx->eflags.u32 = uVal;
5216 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5217 {
5218 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5219 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5220
5221 pMixedCtx->eflags.Bits.u1VM = 0;
5222 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5223 }
5224
5225 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_RFLAGS;
5226 }
5227 return VINF_SUCCESS;
5228}
5229
5230
5231/**
5232 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5233 * guest-CPU context.
5234 */
5235DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236{
5237 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5238 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5239 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5240 return rc;
5241}
5242
5243
5244/**
5245 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5246 * from the guest-state area in the VMCS.
5247 *
5248 * @param pVCpu Pointer to the VMCPU.
5249 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5250 * out-of-sync. Make sure to update the required fields
5251 * before using them.
5252 *
5253 * @remarks No-long-jump zone!!!
5254 */
5255static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5256{
5257 uint32_t uIntrState = 0;
5258 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5259 AssertRC(rc);
5260
5261 if (!uIntrState)
5262 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5263 else
5264 {
5265 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5266 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5267 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5268 AssertRC(rc);
5269 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5270 AssertRC(rc);
5271
5272 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5273 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5274 }
5275}
5276
5277
5278/**
5279 * Saves the guest's activity state.
5280 *
5281 * @returns VBox status code.
5282 * @param pVCpu Pointer to the VMCPU.
5283 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5284 * out-of-sync. Make sure to update the required fields
5285 * before using them.
5286 *
5287 * @remarks No-long-jump zone!!!
5288 */
5289static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5290{
5291 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5292 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5293 return VINF_SUCCESS;
5294}
5295
5296
5297/**
5298 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5299 * the current VMCS into the guest-CPU context.
5300 *
5301 * @returns VBox status code.
5302 * @param pVCpu Pointer to the VMCPU.
5303 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5304 * out-of-sync. Make sure to update the required fields
5305 * before using them.
5306 *
5307 * @remarks No-long-jump zone!!!
5308 */
5309static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5310{
5311 int rc = VINF_SUCCESS;
5312 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5313 {
5314 uint32_t u32Val = 0;
5315 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5316 pMixedCtx->SysEnter.cs = u32Val;
5317 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5318 }
5319
5320 uint64_t u64Val = 0;
5321 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5322 {
5323 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
5324 pMixedCtx->SysEnter.eip = u64Val;
5325 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5326 }
5327 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5328 {
5329 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5330 pMixedCtx->SysEnter.esp = u64Val;
5331 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5332 }
5333 return rc;
5334}
5335
5336
5337/**
5338 * Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5339 * context.
5340 *
5341 * @returns VBox status code.
5342 * @param pVCpu Pointer to the VMCPU.
5343 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5344 * out-of-sync. Make sure to update the required fields
5345 * before using them.
5346 *
5347 * @remarks No-long-jump zone!!!
5348 */
5349static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5350{
5351 int rc = VINF_SUCCESS;
5352 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5353 {
5354 uint64_t u64Val = 0;
5355 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5356 pMixedCtx->fs.u64Base = u64Val;
5357 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5358 }
5359 return rc;
5360}
5361
5362
5363/**
5364 * Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5365 * context.
5366 *
5367 * @returns VBox status code.
5368 * @param pVCpu Pointer to the VMCPU.
5369 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5370 * out-of-sync. Make sure to update the required fields
5371 * before using them.
5372 *
5373 * @remarks No-long-jump zone!!!
5374 */
5375static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376{
5377 int rc = VINF_SUCCESS;
5378 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5379 {
5380 uint64_t u64Val = 0;
5381 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5382 pMixedCtx->gs.u64Base = u64Val;
5383 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5384 }
5385 return rc;
5386}
5387
5388
5389/**
5390 * Saves the auto load/store'd guest MSRs from the current VMCS into the
5391 * guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5392 * and TSC_AUX.
5393 *
5394 * @returns VBox status code.
5395 * @param pVCpu Pointer to the VMCPU.
5396 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5397 * out-of-sync. Make sure to update the required fields
5398 * before using them.
5399 *
5400 * @remarks No-long-jump zone!!!
5401 */
5402static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5403{
5404 if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5405 return VINF_SUCCESS;
5406
5407#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5408 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5409 {
5410 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5411 pMsr += i;
5412 switch (pMsr->u32Msr)
5413 {
5414 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5415 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5416 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5417 case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5418 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5419 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5420 default:
5421 {
5422 AssertFailed();
5423 return VERR_HM_UNEXPECTED_LD_ST_MSR;
5424 }
5425 }
5426 }
5427#endif
5428
5429 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5430 return VINF_SUCCESS;
5431}
5432
5433
5434/**
5435 * Saves the guest control registers from the current VMCS into the guest-CPU
5436 * context.
5437 *
5438 * @returns VBox status code.
5439 * @param pVCpu Pointer to the VMCPU.
5440 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5441 * out-of-sync. Make sure to update the required fields
5442 * before using them.
5443 *
5444 * @remarks No-long-jump zone!!!
5445 */
5446static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5447{
5448 /* Guest CR0. Guest FPU. */
5449 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5450 AssertRCReturn(rc, rc);
5451
5452 /* Guest CR4. */
5453 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5454 AssertRCReturn(rc, rc);
5455
5456 /* Guest CR2 - updated always during the world-switch or in #PF. */
5457 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5458 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5459 {
5460 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5461 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5462
5463 PVM pVM = pVCpu->CTX_SUFF(pVM);
5464 if ( pVM->hm.s.vmx.fUnrestrictedGuest
5465 || ( pVM->hm.s.fNestedPaging
5466 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5467 {
5468 uint64_t u64Val = 0;
5469 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5470 if (pMixedCtx->cr3 != u64Val)
5471 {
5472 CPUMSetGuestCR3(pVCpu, u64Val);
5473 if (VMMRZCallRing3IsEnabled(pVCpu))
5474 {
5475 PGMUpdateCR3(pVCpu, u64Val);
5476 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5477 }
5478 else
5479 {
5480 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5481 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5482 }
5483 }
5484
5485 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5486 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5487 {
5488 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5489 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5490 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5491 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5492
5493 if (VMMRZCallRing3IsEnabled(pVCpu))
5494 {
5495 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5496 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5497 }
5498 else
5499 {
5500 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5501 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5502 }
5503 }
5504 }
5505
5506 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_CR3;
5507 }
5508
5509 /*
5510 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5511 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5512 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5513 *
5514 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5515 */
5516 if (VMMRZCallRing3IsEnabled(pVCpu))
5517 {
5518 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5519 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5520
5521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5522 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5523
5524 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5525 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5526 }
5527
5528 return rc;
5529}
5530
5531
5532/**
5533 * Reads a guest segment register from the current VMCS into the guest-CPU
5534 * context.
5535 *
5536 * @returns VBox status code.
5537 * @param pVCpu Pointer to the VMCPU.
5538 * @param idxSel Index of the selector in the VMCS.
5539 * @param idxLimit Index of the segment limit in the VMCS.
5540 * @param idxBase Index of the segment base in the VMCS.
5541 * @param idxAccess Index of the access rights of the segment in the VMCS.
5542 * @param pSelReg Pointer to the segment selector.
5543 *
5544 * @remarks No-long-jump zone!!!
5545 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5546 * macro as that takes care of whether to read from the VMCS cache or
5547 * not.
5548 */
5549DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5550 PCPUMSELREG pSelReg)
5551{
5552 uint32_t u32Val = 0;
5553 int rc = VMXReadVmcs32(idxSel, &u32Val);
5554 AssertRCReturn(rc, rc);
5555 pSelReg->Sel = (uint16_t)u32Val;
5556 pSelReg->ValidSel = (uint16_t)u32Val;
5557 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5558
5559 rc = VMXReadVmcs32(idxLimit, &u32Val);
5560 AssertRCReturn(rc, rc);
5561 pSelReg->u32Limit = u32Val;
5562
5563 uint64_t u64Val = 0;
5564 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5565 AssertRCReturn(rc, rc);
5566 pSelReg->u64Base = u64Val;
5567
5568 rc = VMXReadVmcs32(idxAccess, &u32Val);
5569 AssertRCReturn(rc, rc);
5570 pSelReg->Attr.u = u32Val;
5571
5572 /*
5573 * If VT-x marks the segment as unusable, most other bits remain undefined:
5574 * - For CS the L, D and G bits have meaning.
5575 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
5576 * - For the remaining data segments no bits are defined.
5577 *
5578 * The present bit and the unusable bit has been observed to be set at the
5579 * same time (the selector was supposed to invalid as we started executing
5580 * a V8086 interrupt in ring-0).
5581 *
5582 * What should be important for the rest of the VBox code that the P bit is
5583 * cleared. Some of the other VBox code recognizes the unusable bit, but
5584 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
5585 * safe side here, we'll strip off P and other bits we don't care about. If
5586 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
5587 *
5588 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5589 */
5590 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
5591 {
5592 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5593
5594 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
5595 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
5596 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
5597
5598 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
5599#ifdef DEBUG_bird
5600 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
5601 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
5602 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
5603#endif
5604 }
5605 return VINF_SUCCESS;
5606}
5607
5608
5609#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5610# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5611 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5612 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5613#else
5614# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5615 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5616 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5617#endif
5618
5619
5620/**
5621 * Saves the guest segment registers from the current VMCS into the guest-CPU
5622 * context.
5623 *
5624 * @returns VBox status code.
5625 * @param pVCpu Pointer to the VMCPU.
5626 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5627 * out-of-sync. Make sure to update the required fields
5628 * before using them.
5629 *
5630 * @remarks No-long-jump zone!!!
5631 */
5632static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5633{
5634 /* Guest segment registers. */
5635 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5636 {
5637 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5638 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5639 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5640 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5641 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5642 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5643 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5644
5645 /* Restore segment attributes for real-on-v86 mode hack. */
5646 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5647 {
5648 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
5649 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
5650 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
5651 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
5652 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
5653 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
5654 }
5655 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5656 }
5657
5658 return VINF_SUCCESS;
5659}
5660
5661
5662/**
5663 * Saves the guest descriptor table registers and task register from the current
5664 * VMCS into the guest-CPU context.
5665 *
5666 * @returns VBox status code.
5667 * @param pVCpu Pointer to the VMCPU.
5668 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5669 * out-of-sync. Make sure to update the required fields
5670 * before using them.
5671 *
5672 * @remarks No-long-jump zone!!!
5673 */
5674static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5675{
5676 int rc = VINF_SUCCESS;
5677
5678 /* Guest LDTR. */
5679 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5680 {
5681 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5682 AssertRCReturn(rc, rc);
5683 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_LDTR;
5684 }
5685
5686 /* Guest GDTR. */
5687 uint64_t u64Val = 0;
5688 uint32_t u32Val = 0;
5689 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5690 {
5691 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5692 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5693 pMixedCtx->gdtr.pGdt = u64Val;
5694 pMixedCtx->gdtr.cbGdt = u32Val;
5695 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_GDTR;
5696 }
5697
5698 /* Guest IDTR. */
5699 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5700 {
5701 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5702 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5703 pMixedCtx->idtr.pIdt = u64Val;
5704 pMixedCtx->idtr.cbIdt = u32Val;
5705 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_IDTR;
5706 }
5707
5708 /* Guest TR. */
5709 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5710 {
5711 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5712 AssertRCReturn(rc, rc);
5713
5714 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5715 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5716 {
5717 rc = VMXLOCAL_READ_SEG(TR, tr);
5718 AssertRCReturn(rc, rc);
5719 }
5720 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_TR;
5721 }
5722 return rc;
5723}
5724
5725#undef VMXLOCAL_READ_SEG
5726
5727
5728/**
5729 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
5730 * context.
5731 *
5732 * @returns VBox status code.
5733 * @param pVCpu Pointer to the VMCPU.
5734 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5735 * out-of-sync. Make sure to update the required fields
5736 * before using them.
5737 *
5738 * @remarks No-long-jump zone!!!
5739 */
5740static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5741{
5742 if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5743 {
5744 if (!CPUMIsHyperDebugStateActive(pVCpu))
5745 {
5746 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5747 uint32_t u32Val;
5748 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5749 pMixedCtx->dr[7] = u32Val;
5750 }
5751
5752 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_DEBUG;
5753 }
5754 return VINF_SUCCESS;
5755}
5756
5757
5758/**
5759 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
5760 *
5761 * @returns VBox status code.
5762 * @param pVCpu Pointer to the VMCPU.
5763 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5764 * out-of-sync. Make sure to update the required fields
5765 * before using them.
5766 *
5767 * @remarks No-long-jump zone!!!
5768 */
5769static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5770{
5771 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5772 pVCpu->hm.s.vmx.fUpdatedGuestState |= HMVMX_UPDATED_GUEST_APIC_STATE;
5773 return VINF_SUCCESS;
5774}
5775
5776
5777/**
5778 * Saves the entire guest state from the currently active VMCS into the
5779 * guest-CPU context. This essentially VMREADs all guest-data.
5780 *
5781 * @returns VBox status code.
5782 * @param pVCpu Pointer to the VMCPU.
5783 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5784 * out-of-sync. Make sure to update the required fields
5785 * before using them.
5786 */
5787static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5788{
5789 Assert(pVCpu);
5790 Assert(pMixedCtx);
5791
5792 if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5793 return VINF_SUCCESS;
5794
5795 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5796 there is no real need to. */
5797 if (VMMRZCallRing3IsEnabled(pVCpu))
5798 VMMR0LogFlushDisable(pVCpu);
5799 else
5800 Assert(VMMR0IsLogFlushDisabled(pVCpu));
5801 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
5802
5803 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5804 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5805
5806 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5807 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5808
5809 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5810 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5811
5812 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5813 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5814
5815 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
5816 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5817
5818 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5819 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5820
5821 rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5822 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5823
5824 rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5825 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5826
5827 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5828 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5829
5830 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5831 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5832
5833 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5834 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5835
5836 AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5837 ("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5838
5839 if (VMMRZCallRing3IsEnabled(pVCpu))
5840 VMMR0LogFlushEnable(pVCpu);
5841
5842 return rc;
5843}
5844
5845
5846/**
5847 * Check per-VM and per-VCPU force flag actions that require us to go back to
5848 * ring-3 for one reason or another.
5849 *
5850 * @returns VBox status code (information status code included).
5851 * @retval VINF_SUCCESS if we don't have any actions that require going back to
5852 * ring-3.
5853 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5854 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5855 * interrupts)
5856 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5857 * all EMTs to be in ring-3.
5858 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5859 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5860 * to the EM loop.
5861 *
5862 * @param pVM Pointer to the VM.
5863 * @param pVCpu Pointer to the VMCPU.
5864 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865 * out-of-sync. Make sure to update the required fields
5866 * before using them.
5867 */
5868static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5869{
5870 Assert(VMMRZCallRing3IsEnabled(pVCpu));
5871
5872 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
5873 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
5874 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
5875 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
5876 {
5877 /* We need the control registers now, make sure the guest-CPU context is updated. */
5878 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5879 AssertRCReturn(rc3, rc3);
5880
5881 /* Pending HM CR3 sync. */
5882 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5883 {
5884 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5885 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
5886 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
5887 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5888 }
5889
5890 /* Pending HM PAE PDPEs. */
5891 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5892 {
5893 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5894 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5895 }
5896
5897 /* Pending PGM C3 sync. */
5898 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5899 {
5900 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5901 if (rc2 != VINF_SUCCESS)
5902 {
5903 AssertRC(rc2);
5904 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
5905 return rc2;
5906 }
5907 }
5908
5909 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5910 /* -XXX- what was that about single stepping? */
5911 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5912 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5913 {
5914 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5915 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5916 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
5917 return rc2;
5918 }
5919
5920 /* Pending VM request packets, such as hardware interrupts. */
5921 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5922 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5923 {
5924 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5925 return VINF_EM_PENDING_REQUEST;
5926 }
5927
5928 /* Pending PGM pool flushes. */
5929 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5930 {
5931 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5932 return VINF_PGM_POOL_FLUSH_PENDING;
5933 }
5934
5935 /* Pending DMA requests. */
5936 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5937 {
5938 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5939 return VINF_EM_RAW_TO_R3;
5940 }
5941 }
5942
5943 /* Paranoia. */
5944 return VINF_SUCCESS;
5945}
5946
5947
5948/**
5949 * Converts any TRPM trap into a pending HM event. This is typically used when
5950 * entering from ring-3 (not longjmp returns).
5951 *
5952 * @param pVCpu Pointer to the VMCPU.
5953 */
5954static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
5955{
5956 Assert(TRPMHasTrap(pVCpu));
5957 Assert(!pVCpu->hm.s.Event.fPending);
5958
5959 uint8_t uVector;
5960 TRPMEVENT enmTrpmEvent;
5961 RTGCUINT uErrCode;
5962 RTGCUINTPTR GCPtrFaultAddress;
5963 uint8_t cbInstr;
5964
5965 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5966 AssertRC(rc);
5967
5968 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5969 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
5970 if (enmTrpmEvent == TRPM_TRAP)
5971 {
5972 switch (uVector)
5973 {
5974 case X86_XCPT_BP:
5975 case X86_XCPT_OF:
5976 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5977 break;
5978
5979 case X86_XCPT_PF:
5980 case X86_XCPT_DF:
5981 case X86_XCPT_TS:
5982 case X86_XCPT_NP:
5983 case X86_XCPT_SS:
5984 case X86_XCPT_GP:
5985 case X86_XCPT_AC:
5986 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5987 /* no break! */
5988 default:
5989 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5990 break;
5991 }
5992 }
5993 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5994 {
5995 if (uVector == X86_XCPT_NMI)
5996 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5997 else
5998 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5999 }
6000 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6001 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6002 else
6003 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6004
6005 rc = TRPMResetTrap(pVCpu);
6006 AssertRC(rc);
6007 Log4(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6008 u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6009
6010 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6011 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6012}
6013
6014
6015/**
6016 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6017 * VT-x to execute any instruction.
6018 *
6019 * @param pvCpu Pointer to the VMCPU.
6020 */
6021static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6022{
6023 Assert(pVCpu->hm.s.Event.fPending);
6024
6025 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6026 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
6027 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
6028 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6029
6030 /* If a trap was already pending, we did something wrong! */
6031 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6032
6033 TRPMEVENT enmTrapType;
6034 switch (uVectorType)
6035 {
6036 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6037 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6038 enmTrapType = TRPM_HARDWARE_INT;
6039 break;
6040
6041 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6042 enmTrapType = TRPM_SOFTWARE_INT;
6043 break;
6044
6045 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6046 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6047 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6048 enmTrapType = TRPM_TRAP;
6049 break;
6050
6051 default:
6052 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6053 enmTrapType = TRPM_32BIT_HACK;
6054 break;
6055 }
6056
6057 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6058
6059 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6060 AssertRC(rc);
6061
6062 if (fErrorCodeValid)
6063 TRPMSetErrorCode(pVCpu, uErrorCode);
6064
6065 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6066 && uVector == X86_XCPT_PF)
6067 {
6068 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6069 }
6070 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6071 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6072 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6073 {
6074 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6075 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6076 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6077 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6078 }
6079 pVCpu->hm.s.Event.fPending = false;
6080}
6081
6082
6083/**
6084 * Does the necessary state syncing before returning to ring-3 for any reason
6085 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6086 *
6087 * @returns VBox status code.
6088 * @param pVM Pointer to the VM.
6089 * @param pVCpu Pointer to the VMCPU.
6090 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6091 * out-of-sync. Make sure to update the required fields
6092 * before using them.
6093 *
6094 * @remarks No-long-jmp zone!!!
6095 */
6096static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6097{
6098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6099 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6100
6101 RTCPUID idCpu = RTMpCpuId();
6102 Log4Func(("HostCpuId=%u\n", idCpu));
6103
6104 /* Save the guest state if necessary. */
6105 if (pVCpu->hm.s.vmx.fUpdatedGuestState != HMVMX_UPDATED_GUEST_ALL)
6106 {
6107 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6108 AssertRCReturn(rc, rc);
6109 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
6110 }
6111
6112 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6113 if (CPUMIsGuestFPUStateActive(pVCpu))
6114 {
6115 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6116 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6117 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
6118 }
6119
6120 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6121#ifdef VBOX_STRICT
6122 if (CPUMIsHyperDebugStateActive(pVCpu))
6123 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6124#endif
6125 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6126 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
6127 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
6128 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
6129
6130 /* Restore host-state bits that VT-x only restores partially. */
6131 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6132 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6133 {
6134 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6135 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6136 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6137 }
6138
6139 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6140 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6141 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6142 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6143 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6144 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6145 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6146 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6147
6148 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6149
6150 /** @todo This kinda defeats the purpose of having preemption hooks.
6151 * The problem is, deregistering the hooks should be moved to a place that
6152 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6153 * context.
6154 */
6155 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6156 {
6157 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6158 AssertRCReturn(rc, rc);
6159
6160 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6161 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6162 }
6163 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6164 NOREF(idCpu);
6165
6166 return VINF_SUCCESS;
6167}
6168
6169
6170/**
6171 * Leaves the VT-x session.
6172 *
6173 * @returns VBox status code.
6174 * @param pVM Pointer to the VM.
6175 * @param pVCpu Pointer to the VMCPU.
6176 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6177 * out-of-sync. Make sure to update the required fields
6178 * before using them.
6179 *
6180 * @remarks No-long-jmp zone!!!
6181 */
6182DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6183{
6184 HM_DISABLE_PREEMPT_IF_NEEDED();
6185 HMVMX_ASSERT_CPU_SAFE();
6186 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6187 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6188
6189 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6190 and done this from the VMXR0ThreadCtxCallback(). */
6191 if (!pVCpu->hm.s.fLeaveDone)
6192 {
6193 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6194 AssertRCReturn(rc2, rc2);
6195 pVCpu->hm.s.fLeaveDone = true;
6196 }
6197
6198 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6199 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6200 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6201 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6202 VMMR0ThreadCtxHooksDeregister(pVCpu);
6203
6204 /* Leave HM context. This takes care of local init (term). */
6205 int rc = HMR0LeaveCpu(pVCpu);
6206
6207 HM_RESTORE_PREEMPT_IF_NEEDED();
6208
6209 return rc;
6210}
6211
6212
6213/**
6214 * Does the necessary state syncing before doing a longjmp to ring-3.
6215 *
6216 * @returns VBox status code.
6217 * @param pVM Pointer to the VM.
6218 * @param pVCpu Pointer to the VMCPU.
6219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6220 * out-of-sync. Make sure to update the required fields
6221 * before using them.
6222 *
6223 * @remarks No-long-jmp zone!!!
6224 */
6225DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6226{
6227 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6228}
6229
6230
6231/**
6232 * Take necessary actions before going back to ring-3.
6233 *
6234 * An action requires us to go back to ring-3. This function does the necessary
6235 * steps before we can safely return to ring-3. This is not the same as longjmps
6236 * to ring-3, this is voluntary and prepares the guest so it may continue
6237 * executing outside HM (recompiler/IEM).
6238 *
6239 * @returns VBox status code.
6240 * @param pVM Pointer to the VM.
6241 * @param pVCpu Pointer to the VMCPU.
6242 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6243 * out-of-sync. Make sure to update the required fields
6244 * before using them.
6245 * @param rcExit The reason for exiting to ring-3. Can be
6246 * VINF_VMM_UNKNOWN_RING3_CALL.
6247 */
6248static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
6249{
6250 Assert(pVM);
6251 Assert(pVCpu);
6252 Assert(pMixedCtx);
6253 HMVMX_ASSERT_PREEMPT_SAFE();
6254
6255 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
6256 {
6257 /* We've done what is required in hmR0VmxExitErrInvalidGuestState(). We're not going to continue guest execution... */
6258 return VINF_SUCCESS;
6259 }
6260 else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6261 {
6262 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6263 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6264 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6265 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6266 return VINF_SUCCESS;
6267 }
6268
6269 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6270 VMMRZCallRing3Disable(pVCpu);
6271 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
6272
6273 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6274 if (pVCpu->hm.s.Event.fPending)
6275 {
6276 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6277 Assert(!pVCpu->hm.s.Event.fPending);
6278 }
6279
6280 /* Save guest state and restore host state bits. */
6281 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6282 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6283
6284 /* Sync recompiler state. */
6285 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6286 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6287 | CPUM_CHANGED_LDTR
6288 | CPUM_CHANGED_GDTR
6289 | CPUM_CHANGED_IDTR
6290 | CPUM_CHANGED_TR
6291 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6292 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
6293 if ( pVM->hm.s.fNestedPaging
6294 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6295 {
6296 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6297 }
6298
6299 /*
6300 * Clear the X86_EFL_TF if necessary .
6301 */
6302 if (pVCpu->hm.s.fClearTrapFlag)
6303 {
6304 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6305 pMixedCtx->eflags.Bits.u1TF = 0;
6306 pVCpu->hm.s.fClearTrapFlag = false;
6307 }
6308 /** @todo there seems to be issues with the resume flag when the monitor trap
6309 * flag is pending without being used. Seen early in bios init when
6310 * accessing APIC page in prot mode. */
6311
6312 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6313 if (rcExit != VINF_EM_RAW_INTERRUPT)
6314 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
6315
6316 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6317
6318 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6319 VMMRZCallRing3RemoveNotification(pVCpu);
6320 VMMRZCallRing3Enable(pVCpu);
6321
6322 return rc;
6323}
6324
6325
6326/**
6327 * VMMRZCallRing3() callback wrapper which saves the guest state before we
6328 * longjump to ring-3 and possibly get preempted.
6329 *
6330 * @returns VBox status code.
6331 * @param pVCpu Pointer to the VMCPU.
6332 * @param enmOperation The operation causing the ring-3 longjump.
6333 * @param pvUser Opaque pointer to the guest-CPU context. The data
6334 * may be out-of-sync. Make sure to update the required
6335 * fields before using them.
6336 *
6337 * @remarks Must never be called with @a enmOperation ==
6338 * VMMCALLRING3_VM_R0_ASSERTION. We can't assert it here because if it
6339 * it -does- get called with VMMCALLRING3_VM_R0_ASSERTION, we'll end up
6340 * with an infinite recursion.
6341 */
6342DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
6343{
6344 /* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion. */
6345 Assert(pVCpu);
6346 Assert(pvUser);
6347 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6348 HMVMX_ASSERT_PREEMPT_SAFE();
6349
6350 VMMRZCallRing3Disable(pVCpu);
6351 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6352
6353 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
6354 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
6355 AssertRCReturn(rc, rc);
6356
6357 VMMRZCallRing3Enable(pVCpu);
6358 return VINF_SUCCESS;
6359}
6360
6361
6362/**
6363 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
6364 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
6365 *
6366 * @param pVCpu Pointer to the VMCPU.
6367 */
6368DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
6369{
6370 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6371 {
6372 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
6373 {
6374 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
6375 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
6376 AssertRC(rc);
6377 }
6378 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
6379}
6380
6381
6382/**
6383 * Evaluates the event to be delivered to the guest and sets it as the pending
6384 * event.
6385 *
6386 * @param pVCpu Pointer to the VMCPU.
6387 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6388 * out-of-sync. Make sure to update the required fields
6389 * before using them.
6390 */
6391static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6392{
6393 Assert(!pVCpu->hm.s.Event.fPending);
6394
6395 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6396 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6397 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6398 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6399
6400 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6401 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6402 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6403 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6404 Assert(!TRPMHasTrap(pVCpu));
6405
6406 /** @todo SMI. SMIs take priority over NMIs. */
6407 if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
6408 {
6409 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6410 if ( !fBlockMovSS
6411 && !fBlockSti)
6412 {
6413 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
6414 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
6415 uint32_t u32IntrInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
6416 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6417
6418 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
6419 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
6420 }
6421 else
6422 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6423 }
6424 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
6425 && !pVCpu->hm.s.fSingleInstruction)
6426 {
6427 /*
6428 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
6429 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt which is why it is
6430 * evaluated here and not set as pending, solely based on the force-flags.
6431 */
6432 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6433 AssertRC(rc);
6434 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6435 if ( !fBlockInt
6436 && !fBlockSti
6437 && !fBlockMovSS)
6438 {
6439 uint8_t u8Interrupt;
6440 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
6441 if (RT_SUCCESS(rc))
6442 {
6443 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
6444 uint32_t u32IntrInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
6445 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6446
6447 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
6448 }
6449 else
6450 {
6451 /** @todo Does this actually happen? If not turn it into an assertion. */
6452 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
6453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6454 }
6455 }
6456 else
6457 hmR0VmxSetIntWindowExitVmcs(pVCpu);
6458 }
6459}
6460
6461
6462/**
6463 * Injects any pending events into the guest if the guest is in a state to
6464 * receive them.
6465 *
6466 * @returns VBox status code (informational status codes included).
6467 * @param pVCpu Pointer to the VMCPU.
6468 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6469 * out-of-sync. Make sure to update the required fields
6470 * before using them.
6471 *
6472 * @remarks No-long-jump zone!!!
6473 */
6474static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6475{
6476 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
6477 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
6478 bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6479 bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6480
6481 Assert(!fBlockSti || (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
6482 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
6483 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
6484 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
6485 Assert(!TRPMHasTrap(pVCpu));
6486
6487 int rc = VINF_SUCCESS;
6488 if (pVCpu->hm.s.Event.fPending)
6489 {
6490#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
6491 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
6492 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6493 {
6494 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6495 AssertRCReturn(rc, rc);
6496 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
6497 Assert(!fBlockInt);
6498 Assert(!fBlockSti);
6499 Assert(!fBlockMovSS);
6500 }
6501 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6502 {
6503 Assert(!fBlockSti);
6504 Assert(!fBlockMovSS);
6505 }
6506#endif
6507 Log4(("Injecting pending event vcpu[%RU32] u64IntrInfo=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo));
6508 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
6509 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
6510 AssertRCReturn(rc, rc);
6511
6512 pVCpu->hm.s.Event.fPending = false;
6513
6514 /* Update the interruptibility-state as it could have been changed by
6515 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
6516 fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6517 fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6518
6519#ifdef VBOX_WITH_STATISTICS
6520 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
6521 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
6522 else
6523 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
6524#endif
6525 }
6526
6527 /* Delivery pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
6528 int rc2 = VINF_SUCCESS;
6529 if ( fBlockSti
6530 || fBlockMovSS)
6531 {
6532 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
6533 {
6534 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6535 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6536 {
6537 /*
6538 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
6539 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
6540 * See Intel spec. 27.3.4 "Saving Non-Register State".
6541 */
6542 rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6543 AssertRCReturn(rc, rc);
6544 }
6545 }
6546 else
6547 {
6548 /* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6549 delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6550 uIntrState = 0;
6551 }
6552 }
6553
6554 /*
6555 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6556 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6557 */
6558 rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6559 AssertRC(rc2);
6560
6561 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6562 return rc;
6563}
6564
6565
6566/**
6567 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6568 *
6569 * @param pVCpu Pointer to the VMCPU.
6570 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6571 * out-of-sync. Make sure to update the required fields
6572 * before using them.
6573 */
6574DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6575{
6576 uint32_t u32IntrInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
6577 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6578}
6579
6580
6581/**
6582 * Injects a double-fault (#DF) exception into the VM.
6583 *
6584 * @returns VBox status code (informational status code included).
6585 * @param pVCpu Pointer to the VMCPU.
6586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6587 * out-of-sync. Make sure to update the required fields
6588 * before using them.
6589 */
6590DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6591{
6592 uint32_t u32IntrInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6593 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6594 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6595 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
6596 puIntrState);
6597}
6598
6599
6600/**
6601 * Sets a debug (#DB) exception as pending-for-injection into the VM.
6602 *
6603 * @param pVCpu Pointer to the VMCPU.
6604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6605 * out-of-sync. Make sure to update the required fields
6606 * before using them.
6607 */
6608DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6609{
6610 uint32_t u32IntrInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
6611 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6612 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6613}
6614
6615
6616/**
6617 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
6618 *
6619 * @param pVCpu Pointer to the VMCPU.
6620 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6621 * out-of-sync. Make sure to update the required fields
6622 * before using them.
6623 * @param cbInstr The value of RIP that is to be pushed on the guest
6624 * stack.
6625 */
6626DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6627{
6628 uint32_t u32IntrInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
6629 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6630 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6631}
6632
6633
6634/**
6635 * Injects a general-protection (#GP) fault into the VM.
6636 *
6637 * @returns VBox status code (informational status code included).
6638 * @param pVCpu Pointer to the VMCPU.
6639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6640 * out-of-sync. Make sure to update the required fields
6641 * before using them.
6642 * @param u32ErrorCode The error code associated with the #GP.
6643 */
6644DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6645 uint32_t *puIntrState)
6646{
6647 uint32_t u32IntrInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
6648 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6649 if (fErrorCodeValid)
6650 u32IntrInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6651 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
6652 puIntrState);
6653}
6654
6655
6656/**
6657 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
6658 *
6659 * @param pVCpu Pointer to the VMCPU.
6660 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6661 * out-of-sync. Make sure to update the required fields
6662 * before using them.
6663 * @param uVector The software interrupt vector number.
6664 * @param cbInstr The value of RIP that is to be pushed on the guest
6665 * stack.
6666 */
6667DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6668{
6669 uint32_t u32IntrInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6670 if ( uVector == X86_XCPT_BP
6671 || uVector == X86_XCPT_OF)
6672 {
6673 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6674 }
6675 else
6676 u32IntrInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6677 hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
6678}
6679
6680
6681/**
6682 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6683 * stack.
6684 *
6685 * @returns VBox status code (information status code included).
6686 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6687 * @param pVM Pointer to the VM.
6688 * @param pMixedCtx Pointer to the guest-CPU context.
6689 * @param uValue The value to push to the guest stack.
6690 */
6691DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6692{
6693 /*
6694 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6695 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6696 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6697 */
6698 if (pMixedCtx->sp == 1)
6699 return VINF_EM_RESET;
6700 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6701 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6702 AssertRCReturn(rc, rc);
6703 return rc;
6704}
6705
6706
6707/**
6708 * Injects an event into the guest upon VM-entry by updating the relevant fields
6709 * in the VM-entry area in the VMCS.
6710 *
6711 * @returns VBox status code (informational error codes included).
6712 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6713 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6714 *
6715 * @param pVCpu Pointer to the VMCPU.
6716 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6717 * be out-of-sync. Make sure to update the required
6718 * fields before using them.
6719 * @param u64IntrInfo The VM-entry interruption-information field.
6720 * @param cbInstr The VM-entry instruction length in bytes (for
6721 * software interrupts, exceptions and privileged
6722 * software exceptions).
6723 * @param u32ErrCode The VM-entry exception error code.
6724 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6725 * @param puIntrState Pointer to the current guest interruptibility-state.
6726 * This interruptibility-state will be updated if
6727 * necessary. This cannot not be NULL.
6728 *
6729 * @remarks No-long-jump zone!!!
6730 * @remarks Requires CR0!
6731 */
6732static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6733 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6734{
6735 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6736 AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6737 Assert(puIntrState);
6738 uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6739
6740 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6741 const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6742
6743#ifdef VBOX_STRICT
6744 /* Validate the error-code-valid bit for hardware exceptions. */
6745 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
6746 {
6747 switch (uVector)
6748 {
6749 case X86_XCPT_PF:
6750 case X86_XCPT_DF:
6751 case X86_XCPT_TS:
6752 case X86_XCPT_NP:
6753 case X86_XCPT_SS:
6754 case X86_XCPT_GP:
6755 case X86_XCPT_AC:
6756 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo),
6757 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
6758 /* fallthru */
6759 default:
6760 break;
6761 }
6762 }
6763#endif
6764
6765 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
6766 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6767 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6768
6769 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6770
6771 /* We require CR0 to check if the guest is in real-mode. */
6772 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6773 AssertRCReturn(rc, rc);
6774
6775 /*
6776 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6777 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6778 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6779 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6780 */
6781 if (CPUMIsGuestInRealModeEx(pMixedCtx))
6782 {
6783 PVM pVM = pVCpu->CTX_SUFF(pVM);
6784 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6785 {
6786 Assert(PDMVmmDevHeapIsEnabled(pVM));
6787 Assert(pVM->hm.s.vmx.pRealModeTSS);
6788
6789 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6790 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6791 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6792 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6793 AssertRCReturn(rc, rc);
6794 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6795
6796 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6797 const size_t cbIdtEntry = 4;
6798 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6799 {
6800 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6801 if (uVector == X86_XCPT_DF)
6802 return VINF_EM_RESET;
6803 else if (uVector == X86_XCPT_GP)
6804 {
6805 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6806 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6807 }
6808
6809 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6810 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6811 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
6812 }
6813
6814 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6815 uint16_t uGuestIp = pMixedCtx->ip;
6816 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6817 {
6818 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
6819 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
6820 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6821 }
6822 else if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6823 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6824
6825 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6826 uint16_t offIdtEntry = 0;
6827 RTSEL selIdtEntry = 0;
6828 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6829 rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6830 rc |= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6831 AssertRCReturn(rc, rc);
6832
6833 /* Construct the stack frame for the interrupt/exception handler. */
6834 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6835 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6836 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6837 AssertRCReturn(rc, rc);
6838
6839 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
6840 if (rc == VINF_SUCCESS)
6841 {
6842 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
6843 pMixedCtx->rip = offIdtEntry;
6844 pMixedCtx->cs.Sel = selIdtEntry;
6845 pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6846 if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6847 && uVector == X86_XCPT_PF)
6848 {
6849 pMixedCtx->cr2 = GCPtrFaultAddress;
6850 }
6851
6852 /* If any other guest-state bits are changed here, make sure to update
6853 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
6854 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS
6855 | HM_CHANGED_GUEST_RIP
6856 | HM_CHANGED_GUEST_RFLAGS
6857 | HM_CHANGED_GUEST_RSP;
6858
6859 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6860 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6861 {
6862 Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6863 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6864 Log4(("Clearing inhibition due to STI.\n"));
6865 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6866 }
6867 Log4(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6868 }
6869 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
6870 return rc;
6871 }
6872 else
6873 {
6874 /*
6875 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6876 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6877 */
6878 u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6879 }
6880 }
6881
6882 /* Validate. */
6883 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6884 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6885 Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6886
6887 /* Inject. */
6888 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6889 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6890 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6891 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6892
6893 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6894 && uVector == X86_XCPT_PF)
6895 {
6896 pMixedCtx->cr2 = GCPtrFaultAddress;
6897 }
6898
6899 Log4(("Injecting vcpu[%RU32] u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
6900 u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6901
6902 AssertRCReturn(rc, rc);
6903 return rc;
6904}
6905
6906
6907/**
6908 * Enters the VT-x session.
6909 *
6910 * @returns VBox status code.
6911 * @param pVM Pointer to the VM.
6912 * @param pVCpu Pointer to the VMCPU.
6913 * @param pCpu Pointer to the CPU info struct.
6914 */
6915VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
6916{
6917 AssertPtr(pVM);
6918 AssertPtr(pVCpu);
6919 Assert(pVM->hm.s.vmx.fSupported);
6920 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6921 NOREF(pCpu);
6922
6923 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6924 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
6925
6926#ifdef VBOX_STRICT
6927 /* Make sure we're in VMX root mode. */
6928 RTCCUINTREG u32HostCR4 = ASMGetCR4();
6929 if (!(u32HostCR4 & X86_CR4_VMXE))
6930 {
6931 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6932 return VERR_VMX_X86_CR4_VMXE_CLEARED;
6933 }
6934#endif
6935
6936 /*
6937 * Load the VCPU's VMCS as the current (and active) one.
6938 */
6939 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
6940 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6941 if (RT_FAILURE(rc))
6942 return rc;
6943
6944 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
6945 pVCpu->hm.s.fLeaveDone = false;
6946 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
6947
6948 return VINF_SUCCESS;
6949}
6950
6951
6952/**
6953 * The thread-context callback (only on platforms which support it).
6954 *
6955 * @param enmEvent The thread-context event.
6956 * @param pVCpu Pointer to the VMCPU.
6957 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
6958 * @thread EMT.
6959 */
6960VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
6961{
6962 switch (enmEvent)
6963 {
6964 case RTTHREADCTXEVENT_PREEMPTING:
6965 {
6966 /** @todo Stats. */
6967 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6968 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6969 VMCPU_ASSERT_EMT(pVCpu);
6970
6971 PVM pVM = pVCpu->CTX_SUFF(pVM);
6972 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
6973
6974 /* No longjmps (logger flushes, locks) in this fragile context. */
6975 VMMRZCallRing3Disable(pVCpu);
6976 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
6977
6978 /* Save the guest-state, restore host-state (FPU, debug etc.). */
6979 if (!pVCpu->hm.s.fLeaveDone)
6980 {
6981 hmR0VmxLeave(pVM, pVCpu, pMixedCtx);
6982 pVCpu->hm.s.fLeaveDone = true;
6983 }
6984
6985 /* Leave HM context, takes care of local init (term). */
6986 int rc = HMR0LeaveCpu(pVCpu);
6987 AssertRC(rc); NOREF(rc);
6988
6989 /* Restore longjmp state. */
6990 VMMRZCallRing3Enable(pVCpu);
6991 break;
6992 }
6993
6994 case RTTHREADCTXEVENT_RESUMED:
6995 {
6996 /** @todo Stats. */
6997 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6998 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
6999 VMCPU_ASSERT_EMT(pVCpu);
7000
7001 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7002 VMMRZCallRing3Disable(pVCpu);
7003 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7004
7005 /* Initialize the bare minimum state required for HM. This takes care of
7006 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7007 int rc = HMR0EnterCpu(pVCpu);
7008 AssertRC(rc);
7009 Assert(pVCpu->hm.s.fContextUseFlags & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7010
7011 /* Load the active VMCS as the current one. */
7012 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7013 {
7014 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7015 AssertRC(rc); NOREF(rc);
7016 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7017 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7018 }
7019 pVCpu->hm.s.fLeaveDone = false;
7020 VMMRZCallRing3Enable(pVCpu);
7021 break;
7022 }
7023
7024 default:
7025 break;
7026 }
7027}
7028
7029
7030/**
7031 * Saves the host state in the VMCS host-state.
7032 * Sets up the VM-exit MSR-load area.
7033 *
7034 * The CPU state will be loaded from these fields on every successful VM-exit.
7035 *
7036 * @returns VBox status code.
7037 * @param pVM Pointer to the VM.
7038 * @param pVCpu Pointer to the VMCPU.
7039 *
7040 * @remarks No-long-jump zone!!!
7041 */
7042static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7043{
7044 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7045
7046 if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
7047 return VINF_SUCCESS;
7048
7049 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7050 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7051
7052 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7053 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7054
7055 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7056 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7057
7058 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
7059 return rc;
7060}
7061
7062
7063/**
7064 * Saves the host state in the VMCS host-state.
7065 *
7066 * @returns VBox status code.
7067 * @param pVM Pointer to the VM.
7068 * @param pVCpu Pointer to the VMCPU.
7069 *
7070 * @remarks No-long-jump zone!!!
7071 */
7072VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7073{
7074 AssertPtr(pVM);
7075 AssertPtr(pVCpu);
7076
7077 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7078
7079 /* When thread-context hooks are available, this is done later (when preemption/interrupts are disabled). */
7080 if (!VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7081 {
7082 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7083 return hmR0VmxSaveHostState(pVM, pVCpu);
7084 }
7085 return VINF_SUCCESS;
7086}
7087
7088
7089/**
7090 * Loads the guest state into the VMCS guest-state area. The CPU state will be
7091 * loaded from these fields on every successful VM-entry.
7092 *
7093 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
7094 * Sets up the VM-entry controls.
7095 * Sets up the appropriate VMX non-root function to execute guest code based on
7096 * the guest CPU mode.
7097 *
7098 * @returns VBox status code.
7099 * @param pVM Pointer to the VM.
7100 * @param pVCpu Pointer to the VMCPU.
7101 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7102 * out-of-sync. Make sure to update the required fields
7103 * before using them.
7104 *
7105 * @remarks No-long-jump zone!!!
7106 */
7107static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7108{
7109 AssertPtr(pVM);
7110 AssertPtr(pVCpu);
7111 AssertPtr(pMixedCtx);
7112 HMVMX_ASSERT_PREEMPT_SAFE();
7113
7114#ifdef LOG_ENABLED
7115 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
7116 * probably not initialized yet? Anyway this will do for now.
7117 *
7118 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
7119 * interface and disable ring-3 calls when thread-context hooks are not
7120 * available. */
7121 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
7122 VMMR0LogFlushDisable(pVCpu);
7123#endif
7124
7125 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7126
7127 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
7128
7129 /* Determine real-on-v86 mode. */
7130 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7131 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
7132 && CPUMIsGuestInRealModeEx(pMixedCtx))
7133 {
7134 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7135 }
7136
7137 /*
7138 * Load the guest-state into the VMCS.
7139 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7140 * Ideally, assert that the cross-dependent bits are up to date at the point of using it.
7141 */
7142 int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
7143 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7144
7145 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
7146 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7147
7148 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
7149 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7150
7151 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
7152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7153
7154 /* Assumes CR0 is up-to-date (strict builds require CR0 for segment register validation checks). */
7155 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7156 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7157
7158 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
7159 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7160
7161 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
7162 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7163
7164 /*
7165 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
7166 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
7167 */
7168 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7169 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7170
7171 rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
7172 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7173
7174 /* Clear any unused and reserved bits. */
7175 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
7176
7177#ifdef LOG_ENABLED
7178 /* Only reenable log-flushing if the caller has it enabled. */
7179 if (!fCallerDisabledLogFlush)
7180 VMMR0LogFlushEnable(pVCpu);
7181#endif
7182
7183 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
7184 return rc;
7185}
7186
7187
7188/**
7189 * Loads the state shared between the host and guest into the VMCS.
7190 *
7191 * @param pVM Pointer to the VM.
7192 * @param pVCpu Pointer to the VMCPU.
7193 * @param pCtx Pointer to the guest-CPU context.
7194 *
7195 * @remarks No-long-jump zone!!!
7196 */
7197static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7198{
7199 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7200 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7201
7202 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
7203 {
7204 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
7205 AssertRC(rc);
7206 }
7207
7208 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
7209 {
7210 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
7211 AssertRC(rc);
7212
7213 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
7214 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
7215 {
7216 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
7217 AssertRC(rc);
7218 }
7219 }
7220
7221 AssertMsg(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE), ("fContextUseFlags=%#x\n",
7222 pVCpu->hm.s.fContextUseFlags));
7223}
7224
7225
7226/**
7227 * Worker for loading the guest-state bits in the inner VT-x execution loop.
7228 *
7229 * @param pVM Pointer to the VM.
7230 * @param pVCpu Pointer to the VMCPU.
7231 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7232 * out-of-sync. Make sure to update the required fields
7233 * before using them.
7234 */
7235DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7236{
7237 HMVMX_ASSERT_PREEMPT_SAFE();
7238
7239 Log5(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
7240#ifdef HMVMX_SYNC_FULL_GUEST_STATE
7241 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
7242#endif
7243
7244 if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
7245 {
7246 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
7247 AssertRC(rc);
7248 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
7249 }
7250 else if (pVCpu->hm.s.fContextUseFlags)
7251 {
7252 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
7253 AssertRC(rc);
7254 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
7255 }
7256
7257 /* All the guest state bits should be loaded except maybe the host context and shared host/guest bits. */
7258 AssertMsg( !(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST)
7259 || !(pVCpu->hm.s.fContextUseFlags & ~(HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE)),
7260 ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7261
7262#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
7263 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
7264 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
7265 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
7266#endif
7267}
7268
7269
7270/**
7271 * Does the preparations before executing guest code in VT-x.
7272 *
7273 * This may cause longjmps to ring-3 and may even result in rescheduling to the
7274 * recompiler. We must be cautious what we do here regarding committing
7275 * guest-state information into the VMCS assuming we assuredly execute the
7276 * guest in VT-x. If we fall back to the recompiler after updating the VMCS and
7277 * clearing the common-state (TRPM/forceflags), we must undo those changes so
7278 * that the recompiler can (and should) use them when it resumes guest
7279 * execution. Otherwise such operations must be done when we can no longer
7280 * exit to ring-3.
7281 *
7282 * @returns VBox status code (informational status codes included).
7283 * @retval VINF_SUCCESS if we can proceed with running the guest.
7284 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
7285 * into the guest.
7286 * @retval VINF_* scheduling changes, we have to go back to ring-3.
7287 *
7288 * @param pVM Pointer to the VM.
7289 * @param pVCpu Pointer to the VMCPU.
7290 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7291 * out-of-sync. Make sure to update the required fields
7292 * before using them.
7293 * @param pVmxTransient Pointer to the VMX transient structure.
7294 *
7295 * @remarks Called with preemption disabled.
7296 */
7297static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7298{
7299 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7300
7301#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
7302 PGMRZDynMapFlushAutoSet(pVCpu);
7303#endif
7304
7305 /* Check force flag actions that might require us to go back to ring-3. */
7306 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
7307 if (rc != VINF_SUCCESS)
7308 return rc;
7309
7310#ifndef IEM_VERIFICATION_MODE_FULL
7311 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
7312 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
7313 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
7314 {
7315 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
7316 RTGCPHYS GCPhysApicBase;
7317 GCPhysApicBase = pMixedCtx->msrApicBase;
7318 GCPhysApicBase &= PAGE_BASE_GC_MASK;
7319
7320 /* Unalias any existing mapping. */
7321 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
7322 AssertRCReturn(rc, rc);
7323
7324 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
7325 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
7326 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
7327 AssertRCReturn(rc, rc);
7328
7329 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
7330 }
7331#endif /* !IEM_VERIFICATION_MODE_FULL */
7332
7333 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
7334 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
7335
7336 /*
7337 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
7338 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
7339 */
7340 if (TRPMHasTrap(pVCpu))
7341 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
7342 else if (!pVCpu->hm.s.Event.fPending)
7343 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
7344
7345 /*
7346 * No longjmps to ring-3 from this point on!!!
7347 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
7348 * This also disables flushing of the R0-logger instance (if any).
7349 */
7350 VMMRZCallRing3Disable(pVCpu);
7351
7352 /*
7353 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
7354 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
7355 *
7356 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
7357 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
7358 *
7359 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
7360 * executing guest code.
7361 */
7362 pVmxTransient->uEflags = ASMIntDisableFlags();
7363 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
7364 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7365 {
7366 ASMSetFlags(pVmxTransient->uEflags);
7367 VMMRZCallRing3Enable(pVCpu);
7368 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7369 return VINF_EM_RAW_TO_R3;
7370 }
7371 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
7372 {
7373 ASMSetFlags(pVmxTransient->uEflags);
7374 VMMRZCallRing3Enable(pVCpu);
7375 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
7376 return VINF_EM_RAW_INTERRUPT;
7377 }
7378
7379 /*
7380 * Event injection might result in triple-faulting the VM (real-on-v86 case), which is why it's
7381 * done here and not in hmR0VmxPreRunGuestCommitted() which doesn't expect failures.
7382 */
7383 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
7384 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7385 {
7386 ASMSetFlags(pVmxTransient->uEflags);
7387 VMMRZCallRing3Enable(pVCpu);
7388 return rc;
7389 }
7390
7391 return VINF_SUCCESS;
7392}
7393
7394
7395/**
7396 * Prepares to run guest code in VT-x and we've committed to doing so. This
7397 * means there is no backing out to ring-3 or anywhere else at this
7398 * point.
7399 *
7400 * @param pVM Pointer to the VM.
7401 * @param pVCpu Pointer to the VMCPU.
7402 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7403 * out-of-sync. Make sure to update the required fields
7404 * before using them.
7405 * @param pVmxTransient Pointer to the VMX transient structure.
7406 *
7407 * @remarks Called with preemption disabled.
7408 * @remarks No-long-jump zone!!!
7409 */
7410static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7411{
7412 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7413 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7414 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7415
7416 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7417 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
7418
7419 /*
7420 * Load the host state bits as we may've been preempted (only happens when
7421 * thread-context hooks are used).
7422 */
7423 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
7424 {
7425 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7426 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
7427 AssertRC(rc);
7428 }
7429 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
7430
7431 /*
7432 * If we are injecting events to a real-on-v86 mode guest, we may have to update
7433 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
7434 * Reload only the necessary state, the assertion will catch if other parts of the code
7435 * change.
7436 */
7437 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7438 {
7439 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
7440 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
7441 }
7442
7443 /* Load the state shared between host and guest (FPU, debug). */
7444 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_GUEST_SHARED_STATE)
7445 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
7446 AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags=%#x\n", pVCpu->hm.s.fContextUseFlags));
7447
7448 /*
7449 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
7450 */
7451 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7452 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
7453
7454 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
7455 || HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
7456 {
7457 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
7458 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
7459 }
7460
7461 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
7462 hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
7463
7464 RTCPUID idCurrentCpu = HMR0GetCurrentCpu()->idCpu;
7465 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
7466 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
7467
7468 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
7469
7470 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
7471 to start executing. */
7472
7473#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7474 /*
7475 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
7476 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
7477 */
7478 if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7479 && !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7480 {
7481 pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
7482 uint64_t u64HostTscAux = 0;
7483 int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
7484 AssertRC(rc2);
7485 ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
7486 }
7487#endif
7488}
7489
7490
7491/**
7492 * Performs some essential restoration of state after running guest code in
7493 * VT-x.
7494 *
7495 * @param pVM Pointer to the VM.
7496 * @param pVCpu Pointer to the VMCPU.
7497 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7498 * out-of-sync. Make sure to update the required fields
7499 * before using them.
7500 * @param pVmxTransient Pointer to the VMX transient structure.
7501 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
7502 *
7503 * @remarks Called with interrupts disabled.
7504 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
7505 * unconditionally when it is safe to do so.
7506 */
7507static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
7508{
7509 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7510
7511 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
7512 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
7513 pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
7514 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
7515 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
7516
7517 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
7518 {
7519#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
7520 /* Restore host's TSC_AUX. */
7521 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
7522 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
7523#endif
7524 /** @todo Find a way to fix hardcoding a guestimate. */
7525 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
7526 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
7527 }
7528
7529 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
7530 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
7531 Assert(!(ASMGetFlags() & X86_EFL_IF));
7532 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
7533
7534 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
7535 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
7536 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
7537 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
7538
7539 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
7540 uint32_t uExitReason;
7541 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
7542 rc |= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7543 AssertRC(rc);
7544 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
7545 pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
7546
7547 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
7548 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
7549 {
7550 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
7551 pVmxTransient->fVMEntryFailed));
7552 return;
7553 }
7554
7555 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
7556 {
7557 /* Update the guest interruptibility-state from the VMCS. */
7558 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
7559#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
7560 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7561 AssertRC(rc);
7562#endif
7563 /*
7564 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
7565 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
7566 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
7567 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
7568 */
7569 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7570 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
7571 {
7572 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
7573 AssertRC(rc);
7574 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
7575 }
7576 }
7577}
7578
7579
7580
7581/**
7582 * Runs the guest code using VT-x the normal way.
7583 *
7584 * @returns VBox status code.
7585 * @param pVM Pointer to the VM.
7586 * @param pVCpu Pointer to the VMCPU.
7587 * @param pCtx Pointer to the guest-CPU context.
7588 *
7589 * @note Mostly the same as hmR0VmxRunGuestCodeStep.
7590 * @remarks Called with preemption disabled.
7591 */
7592static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7593{
7594 VMXTRANSIENT VmxTransient;
7595 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7596 int rc = VERR_INTERNAL_ERROR_5;
7597 uint32_t cLoops = 0;
7598
7599 for (;; cLoops++)
7600 {
7601 Assert(!HMR0SuspendPending());
7602 HMVMX_ASSERT_CPU_SAFE();
7603
7604 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7605 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7606 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7607 if (rc != VINF_SUCCESS)
7608 break;
7609
7610 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7611 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7612 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7613
7614 /* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state. */
7615 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7616
7617 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7618 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7619 {
7620 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7621 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7622 return rc;
7623 }
7624
7625 /* Handle the VM-exit. */
7626 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7627 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7628 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7629 HMVMX_START_EXIT_DISPATCH_PROF();
7630#ifdef HMVMX_USE_FUNCTION_TABLE
7631 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7632#else
7633 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7634#endif
7635 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7636 if (rc != VINF_SUCCESS)
7637 break;
7638 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7639 {
7640 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7641 rc = VINF_EM_RAW_INTERRUPT;
7642 break;
7643 }
7644 }
7645
7646 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7647 return rc;
7648}
7649
7650
7651/**
7652 * Single steps guest code using VT-x.
7653 *
7654 * @returns VBox status code.
7655 * @param pVM Pointer to the VM.
7656 * @param pVCpu Pointer to the VMCPU.
7657 * @param pCtx Pointer to the guest-CPU context.
7658 *
7659 * @note Mostly the same as hmR0VmxRunGuestCodeNormal.
7660 * @remarks Called with preemption disabled.
7661 */
7662static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7663{
7664 VMXTRANSIENT VmxTransient;
7665 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
7666 int rc = VERR_INTERNAL_ERROR_5;
7667 uint32_t cLoops = 0;
7668 uint16_t uCsStart = pCtx->cs.Sel;
7669 uint64_t uRipStart = pCtx->rip;
7670
7671 for (;; cLoops++)
7672 {
7673 Assert(!HMR0SuspendPending());
7674 HMVMX_ASSERT_CPU_SAFE();
7675
7676 /* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
7677 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
7678 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
7679 if (rc != VINF_SUCCESS)
7680 break;
7681
7682 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
7683 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
7684 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
7685
7686 /* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state. */
7687 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
7688
7689 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
7690 if (RT_UNLIKELY(rc != VINF_SUCCESS))
7691 {
7692 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
7693 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
7694 return rc;
7695 }
7696
7697 /* Handle the VM-exit. */
7698 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
7699 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
7700 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
7701 HMVMX_START_EXIT_DISPATCH_PROF();
7702#ifdef HMVMX_USE_FUNCTION_TABLE
7703 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
7704#else
7705 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
7706#endif
7707 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
7708 if (rc != VINF_SUCCESS)
7709 break;
7710 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
7711 {
7712 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
7713 rc = VINF_EM_RAW_INTERRUPT;
7714 break;
7715 }
7716
7717 /*
7718 * Did the RIP change, if so, consider it a single step.
7719 * Otherwise, make sure one of the TFs gets set.
7720 */
7721 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
7722 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
7723 AssertRCReturn(rc2, rc2);
7724 if ( pCtx->rip != uRipStart
7725 || pCtx->cs.Sel != uCsStart)
7726 {
7727 rc = VINF_EM_DBG_STEPPED;
7728 break;
7729 }
7730 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
7731 }
7732
7733 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
7734 return rc;
7735}
7736
7737
7738/**
7739 * Runs the guest code using VT-x.
7740 *
7741 * @returns VBox status code.
7742 * @param pVM Pointer to the VM.
7743 * @param pVCpu Pointer to the VMCPU.
7744 * @param pCtx Pointer to the guest-CPU context.
7745 *
7746 * @remarks Called with preemption disabled.
7747 */
7748VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7749{
7750 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7751 Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
7752 HMVMX_ASSERT_PREEMPT_SAFE();
7753
7754 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
7755
7756 int rc;
7757 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
7758 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
7759 else
7760 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
7761
7762 if (rc == VERR_EM_INTERPRETER)
7763 rc = VINF_EM_RAW_EMULATE_INSTR;
7764 else if (rc == VINF_EM_RESET)
7765 rc = VINF_EM_TRIPLE_FAULT;
7766
7767 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
7768 if (RT_FAILURE(rc2))
7769 {
7770 pVCpu->hm.s.u32HMError = rc;
7771 rc = rc2;
7772 }
7773 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
7774 return rc;
7775}
7776
7777
7778#ifndef HMVMX_USE_FUNCTION_TABLE
7779DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
7780{
7781 int rc;
7782 switch (rcReason)
7783 {
7784 case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
7785 case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
7786 case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
7787 case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
7788 case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
7789 case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
7790 case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7791 case VMX_EXIT_XCPT_OR_NMI: rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); break;
7792 case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
7793 case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
7794 case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7795 case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
7796 case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
7797 case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
7798 case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
7799 case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7800 case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
7801 case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
7802 case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
7803 case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
7804 case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
7805 case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
7806 case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
7807 case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
7808 case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
7809 case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7810 case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
7811 case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
7812 case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
7813 case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
7814 case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
7815 case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
7816 case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
7817
7818 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
7819 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
7820 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
7821 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
7822 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7823 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
7824 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
7825 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
7826 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
7827
7828 case VMX_EXIT_VMCALL:
7829 case VMX_EXIT_VMCLEAR:
7830 case VMX_EXIT_VMLAUNCH:
7831 case VMX_EXIT_VMPTRLD:
7832 case VMX_EXIT_VMPTRST:
7833 case VMX_EXIT_VMREAD:
7834 case VMX_EXIT_VMRESUME:
7835 case VMX_EXIT_VMWRITE:
7836 case VMX_EXIT_VMXOFF:
7837 case VMX_EXIT_VMXON:
7838 case VMX_EXIT_INVEPT:
7839 case VMX_EXIT_INVVPID:
7840 case VMX_EXIT_VMFUNC:
7841 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
7842 break;
7843 default:
7844 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7845 break;
7846 }
7847 return rc;
7848}
7849#endif
7850
7851#ifdef DEBUG
7852/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7853# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
7854 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7855
7856# define HMVMX_ASSERT_PREEMPT_CPUID() \
7857 do \
7858 { \
7859 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7860 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7861 } while (0)
7862
7863# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7864 do { \
7865 AssertPtr(pVCpu); \
7866 AssertPtr(pMixedCtx); \
7867 AssertPtr(pVmxTransient); \
7868 Assert(pVmxTransient->fVMEntryFailed == false); \
7869 Assert(ASMIntAreEnabled()); \
7870 HMVMX_ASSERT_PREEMPT_SAFE(); \
7871 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
7872 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
7873 HMVMX_ASSERT_PREEMPT_SAFE(); \
7874 if (VMMR0IsLogFlushDisabled(pVCpu)) \
7875 HMVMX_ASSERT_PREEMPT_CPUID(); \
7876 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7877 } while (0)
7878
7879# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7880 do { \
7881 Log4Func(("\n")); \
7882 } while(0)
7883#else /* Release builds */
7884# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7885# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7886#endif
7887
7888
7889/**
7890 * Advances the guest RIP after reading it from the VMCS.
7891 *
7892 * @returns VBox status code.
7893 * @param pVCpu Pointer to the VMCPU.
7894 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7895 * out-of-sync. Make sure to update the required fields
7896 * before using them.
7897 * @param pVmxTransient Pointer to the VMX transient structure.
7898 *
7899 * @remarks No-long-jump zone!!!
7900 */
7901DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7902{
7903 int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7904 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7905 AssertRCReturn(rc, rc);
7906
7907 pMixedCtx->rip += pVmxTransient->cbInstr;
7908 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
7909 return rc;
7910}
7911
7912
7913/**
7914 * Tries to determine what part of the guest-state VT-x has deemed as invalid
7915 * and update error record fields accordingly.
7916 *
7917 * @return VMX_IGS_* return codes.
7918 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
7919 * wrong with the guest state.
7920 *
7921 * @param pVM Pointer to the VM.
7922 * @param pVCpu Pointer to the VMCPU.
7923 * @param pCtx Pointer to the guest-CPU state.
7924 */
7925static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
7926{
7927#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
7928#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
7929 uError = (err); \
7930 break; \
7931 } else do {} while (0)
7932/* Duplicate of IEM_IS_CANONICAL(). */
7933#define HMVMX_IS_CANONICAL(a_u64Addr) ((uint64_t)(a_u64Addr) + UINT64_C(0x800000000000) < UINT64_C(0x1000000000000))
7934
7935 int rc;
7936 uint64_t u64Val;
7937 uint32_t u32Val;
7938 uint32_t uError = VMX_IGS_ERROR;
7939 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
7940
7941 do
7942 {
7943 /*
7944 * CR0.
7945 */
7946 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7947 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
7948 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
7949 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
7950 if (fUnrestrictedGuest)
7951 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
7952
7953 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7954 AssertRCBreak(rc);
7955 HMVMX_CHECK_BREAK((u32Val & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
7956 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR0), VMX_IGS_CR0_FIXED0);
7957 if ( !fUnrestrictedGuest
7958 && (u32Val & X86_CR0_PG)
7959 && !(u32Val & X86_CR0_PE))
7960 {
7961 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
7962 }
7963
7964 /*
7965 * CR4.
7966 */
7967 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7968 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
7969 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7970 AssertRCBreak(rc);
7971 HMVMX_CHECK_BREAK((u32Val & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
7972 HMVMX_CHECK_BREAK(!(u32Val & ~uZapCR4), VMX_IGS_CR4_FIXED0);
7973
7974 /*
7975 * IA32_DEBUGCTL MSR.
7976 */
7977 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
7978 AssertRCBreak(rc);
7979 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
7980 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
7981 {
7982 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
7983 }
7984 uint64_t u64DebugCtlMsr = u64Val;
7985
7986#ifdef VBOX_STRICT
7987 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
7988 AssertRCBreak(rc);
7989 Assert(u32Val == pVCpu->hm.s.vmx.u32ProcCtls);
7990#endif
7991 bool const fLongModeGuest = !!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
7992
7993 /*
7994 * RIP and RFLAGS.
7995 */
7996 uint32_t u32Eflags;
7997#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
7998 if (HMVMX_IS_64BIT_HOST_MODE())
7999 {
8000 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
8001 AssertRCBreak(rc);
8002 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
8003 if ( !fLongModeGuest
8004 || !pCtx->cs.Attr.n.u1Long)
8005 {
8006 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
8007 }
8008 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
8009 * must be identical if the "IA32e mode guest" VM-entry control is 1
8010 * and CS.L is 1. No check applies if the CPU supports 64
8011 * linear-address bits. */
8012
8013 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
8014 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
8015 AssertRCBreak(rc);
8016 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
8017 VMX_IGS_RFLAGS_RESERVED);
8018 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8019 u32Eflags = u64Val;
8020 }
8021 else
8022#endif
8023 {
8024 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
8025 AssertRCBreak(rc);
8026 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
8027 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
8028 }
8029
8030 if ( fLongModeGuest
8031 || !(pCtx->cr0 & X86_CR0_PE))
8032 {
8033 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
8034 }
8035
8036 uint32_t u32EntryInfo;
8037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
8038 AssertRCBreak(rc);
8039 if ( VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8040 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8041 {
8042 HMVMX_CHECK_BREAK(u32Val & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
8043 }
8044
8045 /*
8046 * 64-bit checks.
8047 */
8048#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8049 if (HMVMX_IS_64BIT_HOST_MODE())
8050 {
8051 if ( fLongModeGuest
8052 && !fUnrestrictedGuest)
8053 {
8054 HMVMX_CHECK_BREAK(CPUMIsGuestPagingEnabledEx(pCtx), VMX_IGS_CR0_PG_LONGMODE);
8055 HMVMX_CHECK_BREAK((pCtx->cr4 & X86_CR4_PAE), VMX_IGS_CR4_PAE_LONGMODE);
8056 }
8057
8058 if ( !fLongModeGuest
8059 && (pCtx->cr4 & X86_CR4_PCIDE))
8060 {
8061 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
8062 }
8063
8064 /** @todo CR3 field must be such that bits 63:52 and bits in the range
8065 * 51:32 beyond the processor's physical-address width are 0. */
8066
8067 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
8068 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
8069 {
8070 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
8071 }
8072
8073 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
8074 AssertRCBreak(rc);
8075 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
8076
8077 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
8078 AssertRCBreak(rc);
8079 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
8080 }
8081#endif
8082
8083 /*
8084 * PERF_GLOBAL MSR.
8085 */
8086 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
8087 {
8088 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
8089 AssertRCBreak(rc);
8090 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
8091 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
8092 }
8093
8094 /*
8095 * PAT MSR.
8096 */
8097 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
8098 {
8099 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
8100 AssertRCBreak(rc);
8101 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
8102 for (unsigned i = 0; i < 8; i++)
8103 {
8104 uint8_t u8Val = (u64Val & 0x7);
8105 if ( u8Val != 0 /* UC */
8106 || u8Val != 1 /* WC */
8107 || u8Val != 4 /* WT */
8108 || u8Val != 5 /* WP */
8109 || u8Val != 6 /* WB */
8110 || u8Val != 7 /* UC- */)
8111 {
8112 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
8113 }
8114 u64Val >>= 3;
8115 }
8116 }
8117
8118 /*
8119 * EFER MSR.
8120 */
8121 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
8122 {
8123 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
8124 AssertRCBreak(rc);
8125 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
8126 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
8127 HMVMX_CHECK_BREAK((u64Val & MSR_K6_EFER_LMA) == (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
8128 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
8129 HMVMX_CHECK_BREAK( fUnrestrictedGuest
8130 || (u64Val & MSR_K6_EFER_LMA) == (pCtx->cr0 & X86_CR0_PG), VMX_IGS_EFER_LMA_PG_MISMATCH);
8131 }
8132
8133 /*
8134 * Segment registers.
8135 */
8136 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8137 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
8138 if (!(u32Eflags & X86_EFL_VM))
8139 {
8140 /* CS */
8141 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
8142 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
8143 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
8144 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
8145 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8146 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
8147 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
8148 /* CS cannot be loaded with NULL in protected mode. */
8149 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
8150 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
8151 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
8152 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
8153 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
8154 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
8155 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
8156 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
8157 else
8158 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
8159
8160 /* SS */
8161 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8162 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
8163 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
8164 if ( !(pCtx->cr0 & X86_CR0_PE)
8165 || pCtx->cs.Attr.n.u4Type == 3)
8166 {
8167 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
8168 }
8169 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
8170 {
8171 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
8172 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
8173 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
8174 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
8175 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
8176 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8177 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
8178 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
8179 }
8180
8181 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
8182 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
8183 {
8184 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
8185 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
8186 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8187 || pCtx->ds.Attr.n.u4Type > 11
8188 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8189 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
8190 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
8191 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
8192 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8193 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
8194 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
8195 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8196 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
8197 }
8198 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
8199 {
8200 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
8201 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
8202 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8203 || pCtx->es.Attr.n.u4Type > 11
8204 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
8205 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
8206 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
8207 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
8208 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8209 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
8210 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
8211 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8212 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
8213 }
8214 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
8215 {
8216 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
8217 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
8218 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8219 || pCtx->fs.Attr.n.u4Type > 11
8220 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
8221 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
8222 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
8223 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
8224 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8225 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
8226 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
8227 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8228 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
8229 }
8230 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
8231 {
8232 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
8233 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
8234 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
8235 || pCtx->gs.Attr.n.u4Type > 11
8236 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
8237 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
8238 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
8239 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
8240 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8241 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
8242 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
8243 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
8244 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
8245 }
8246 /* 64-bit capable CPUs. */
8247#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8248 if (HMVMX_IS_64BIT_HOST_MODE())
8249 {
8250 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8251 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8252 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8253 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8254 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8255 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8256 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8257 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8258 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8259 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8260 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8261 }
8262#endif
8263 }
8264 else
8265 {
8266 /* V86 mode checks. */
8267 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
8268 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8269 {
8270 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
8271 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
8272 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
8273 }
8274 else
8275 {
8276 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
8277 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
8278 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
8279 }
8280
8281 /* CS */
8282 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
8283 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
8284 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
8285 /* SS */
8286 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
8287 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
8288 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
8289 /* DS */
8290 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
8291 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
8292 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
8293 /* ES */
8294 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
8295 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
8296 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
8297 /* FS */
8298 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
8299 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
8300 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
8301 /* GS */
8302 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
8303 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
8304 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
8305 /* 64-bit capable CPUs. */
8306#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8307 if (HMVMX_IS_64BIT_HOST_MODE())
8308 {
8309 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
8310 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
8311 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
8312 || HMVMX_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
8313 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
8314 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
8315 VMX_IGS_LONGMODE_SS_BASE_INVALID);
8316 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
8317 VMX_IGS_LONGMODE_DS_BASE_INVALID);
8318 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
8319 VMX_IGS_LONGMODE_ES_BASE_INVALID);
8320 }
8321#endif
8322 }
8323
8324 /*
8325 * TR.
8326 */
8327 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
8328 /* 64-bit capable CPUs. */
8329#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8330 if (HMVMX_IS_64BIT_HOST_MODE())
8331 {
8332 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
8333 }
8334#endif
8335 if (fLongModeGuest)
8336 {
8337 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
8338 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
8339 }
8340 else
8341 {
8342 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
8343 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
8344 VMX_IGS_TR_ATTR_TYPE_INVALID);
8345 }
8346 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
8347 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
8348 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
8349 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
8350 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8351 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
8352 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
8353 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
8354
8355 /*
8356 * GDTR and IDTR.
8357 */
8358#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8359 if (HMVMX_IS_64BIT_HOST_MODE())
8360 {
8361 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
8362 AssertRCBreak(rc);
8363 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
8364
8365 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
8366 AssertRCBreak(rc);
8367 HMVMX_CHECK_BREAK(HMVMX_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
8368 }
8369#endif
8370
8371 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
8372 AssertRCBreak(rc);
8373 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8374
8375 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
8376 AssertRCBreak(rc);
8377 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
8378
8379 /*
8380 * Guest Non-Register State.
8381 */
8382 /* Activity State. */
8383 uint32_t u32ActivityState;
8384 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
8385 AssertRCBreak(rc);
8386 HMVMX_CHECK_BREAK( !u32ActivityState
8387 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
8388 VMX_IGS_ACTIVITY_STATE_INVALID);
8389 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
8390 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
8391 uint32_t u32IntrState;
8392 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
8393 AssertRCBreak(rc);
8394 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
8395 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8396 {
8397 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
8398 }
8399
8400 /** @todo Activity state and injecting interrupts. Left as a todo since we
8401 * currently don't use activity states but ACTIVE. */
8402
8403 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8404 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
8405
8406 /* Guest interruptibility-state. */
8407 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
8408 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8409 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
8410 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
8411 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8412 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
8413 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
8414 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8415 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
8416 if (VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo))
8417 {
8418 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
8419 {
8420 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8421 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8422 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
8423 }
8424 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8425 {
8426 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
8427 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
8428 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
8429 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
8430 }
8431 }
8432 /** @todo Assumes the processor is not in SMM. */
8433 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8434 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
8435 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
8436 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
8437 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
8438 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
8439 && VMX_ENTRY_INTERRUPTION_INFO_VALID(u32EntryInfo)
8440 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8441 {
8442 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
8443 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
8444 }
8445
8446 /* Pending debug exceptions. */
8447 if (HMVMX_IS_64BIT_HOST_MODE())
8448 {
8449 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
8450 AssertRCBreak(rc);
8451 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
8452 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
8453 u32Val = u64Val; /* For pending debug exceptions checks below. */
8454 }
8455 else
8456 {
8457 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
8458 AssertRCBreak(rc);
8459 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
8460 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
8461 }
8462
8463 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8464 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
8465 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
8466 {
8467 if ( (u32Eflags & X86_EFL_TF)
8468 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8469 {
8470 /* Bit 14 is PendingDebug.BS. */
8471 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
8472 }
8473 if ( !(u32Eflags & X86_EFL_TF)
8474 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
8475 {
8476 /* Bit 14 is PendingDebug.BS. */
8477 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
8478 }
8479 }
8480
8481 /* VMCS link pointer. */
8482 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
8483 AssertRCBreak(rc);
8484 if (u64Val != UINT64_C(0xffffffffffffffff))
8485 {
8486 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
8487 /** @todo Bits beyond the processor's physical-address width MBZ. */
8488 /** @todo 32-bit located in memory referenced by value of this field (as a
8489 * physical address) must contain the processor's VMCS revision ID. */
8490 /** @todo SMM checks. */
8491 }
8492
8493 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries. */
8494
8495 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
8496 if (uError == VMX_IGS_ERROR)
8497 uError = VMX_IGS_REASON_NOT_FOUND;
8498 } while (0);
8499
8500 pVCpu->hm.s.u32HMError = uError;
8501 return uError;
8502
8503#undef HMVMX_ERROR_BREAK
8504#undef HMVMX_CHECK_BREAK
8505#undef HMVMX_IS_CANONICAL
8506}
8507
8508/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8509/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
8510/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8511
8512/** @name VM-exit handlers.
8513 * @{
8514 */
8515
8516/**
8517 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
8518 */
8519HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8520{
8521 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8522 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
8523 /* 32-bit Windows hosts (4 cores) has trouble with this; causes higher interrupt latency. */
8524#if HC_ARCH_BITS == 64
8525 Assert(ASMIntAreEnabled());
8526 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUsePreemptTimer)
8527 return VINF_SUCCESS;
8528#endif
8529 return VINF_EM_RAW_INTERRUPT;
8530}
8531
8532
8533/**
8534 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
8535 */
8536HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8537{
8538 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8539 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
8540
8541 int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8542 AssertRCReturn(rc, rc);
8543
8544 uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
8545 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
8546 && uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8547 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntrInfo));
8548
8549 if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
8550 {
8551 /*
8552 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
8553 * anything we inject is not going to cause a VM-exit directly for the event being injected.
8554 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
8555 *
8556 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
8557 */
8558 VMXDispatchHostNmi();
8559 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmi);
8560 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8561 return VINF_SUCCESS;
8562 }
8563
8564 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8565 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8566 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
8567 {
8568 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8569 return VINF_SUCCESS;
8570 }
8571 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8572 {
8573 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8574 return rc;
8575 }
8576
8577 uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
8578 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
8579 switch (uIntrType)
8580 {
8581 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
8582 Assert(uVector == X86_XCPT_DB || uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8583 /* no break */
8584 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
8585 {
8586 switch (uVector)
8587 {
8588 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
8589 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
8590 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
8591 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
8592 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
8593 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
8594#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8595 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
8596 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8597 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
8598 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8599 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
8600 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8601 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
8602 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8603 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
8604 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
8605#endif
8606 default:
8607 {
8608 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8609 AssertRCReturn(rc, rc);
8610
8611 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
8612 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8613 {
8614 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
8615 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
8616 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8617
8618 rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8619 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8620 AssertRCReturn(rc, rc);
8621 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
8622 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
8623 0 /* GCPtrFaultAddress */);
8624 AssertRCReturn(rc, rc);
8625 }
8626 else
8627 {
8628 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
8629 pVCpu->hm.s.u32HMError = uVector;
8630 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8631 }
8632 break;
8633 }
8634 }
8635 break;
8636 }
8637
8638 default:
8639 {
8640 pVCpu->hm.s.u32HMError = uExitIntrInfo;
8641 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
8642 AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
8643 break;
8644 }
8645 }
8646 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
8647 return rc;
8648}
8649
8650
8651/**
8652 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
8653 */
8654HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8655{
8656 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8657
8658 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
8659 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
8660 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
8661 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8662 AssertRCReturn(rc, rc);
8663
8664 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
8665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
8666 return VINF_SUCCESS;
8667}
8668
8669
8670/**
8671 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
8672 */
8673HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8674{
8675 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8676 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
8677 pVCpu->hm.s.u32HMError = VMX_EXIT_NMI_WINDOW;
8678 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8679}
8680
8681
8682/**
8683 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
8684 */
8685HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8686{
8687 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8688 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
8689 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8690}
8691
8692
8693/**
8694 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
8695 */
8696HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8697{
8698 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8699 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
8700 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8701}
8702
8703
8704/**
8705 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
8706 */
8707HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8708{
8709 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8710 PVM pVM = pVCpu->CTX_SUFF(pVM);
8711 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8712 if (RT_LIKELY(rc == VINF_SUCCESS))
8713 {
8714 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8715 Assert(pVmxTransient->cbInstr == 2);
8716 }
8717 else
8718 {
8719 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
8720 rc = VERR_EM_INTERPRETER;
8721 }
8722 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
8723 return rc;
8724}
8725
8726
8727/**
8728 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
8729 */
8730HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8731{
8732 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8733 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
8734 AssertRCReturn(rc, rc);
8735
8736 if (pMixedCtx->cr4 & X86_CR4_SMXE)
8737 return VINF_EM_RAW_EMULATE_INSTR;
8738
8739 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
8740 pVCpu->hm.s.u32HMError = VMX_EXIT_GETSEC;
8741 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8742}
8743
8744
8745/**
8746 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
8747 */
8748HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8749{
8750 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8751 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8752 AssertRCReturn(rc, rc);
8753
8754 PVM pVM = pVCpu->CTX_SUFF(pVM);
8755 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8756 if (RT_LIKELY(rc == VINF_SUCCESS))
8757 {
8758 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8759 Assert(pVmxTransient->cbInstr == 2);
8760 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8761 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8762 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8763 }
8764 else
8765 {
8766 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
8767 rc = VERR_EM_INTERPRETER;
8768 }
8769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8770 return rc;
8771}
8772
8773
8774/**
8775 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
8776 */
8777HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8778{
8779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8780 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8781 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
8782 AssertRCReturn(rc, rc);
8783
8784 PVM pVM = pVCpu->CTX_SUFF(pVM);
8785 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
8786 if (RT_LIKELY(rc == VINF_SUCCESS))
8787 {
8788 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8789 Assert(pVmxTransient->cbInstr == 3);
8790 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
8791 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
8792 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
8793 }
8794 else
8795 {
8796 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
8797 rc = VERR_EM_INTERPRETER;
8798 }
8799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
8800 return rc;
8801}
8802
8803
8804/**
8805 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
8806 */
8807HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8808{
8809 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8810 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
8811 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
8812 AssertRCReturn(rc, rc);
8813
8814 PVM pVM = pVCpu->CTX_SUFF(pVM);
8815 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8816 if (RT_LIKELY(rc == VINF_SUCCESS))
8817 {
8818 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8819 Assert(pVmxTransient->cbInstr == 2);
8820 }
8821 else
8822 {
8823 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
8824 rc = VERR_EM_INTERPRETER;
8825 }
8826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
8827 return rc;
8828}
8829
8830
8831/**
8832 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
8833 */
8834HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8835{
8836 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8837 PVM pVM = pVCpu->CTX_SUFF(pVM);
8838 Assert(!pVM->hm.s.fNestedPaging);
8839
8840 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8841 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8842 AssertRCReturn(rc, rc);
8843
8844 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
8845 rc = VBOXSTRICTRC_VAL(rc2);
8846 if (RT_LIKELY(rc == VINF_SUCCESS))
8847 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8848 else
8849 {
8850 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
8851 pVmxTransient->uExitQualification, rc));
8852 }
8853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
8854 return rc;
8855}
8856
8857
8858/**
8859 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
8860 */
8861HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8862{
8863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8864 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8865 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8866 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8867 AssertRCReturn(rc, rc);
8868
8869 PVM pVM = pVCpu->CTX_SUFF(pVM);
8870 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8871 if (RT_LIKELY(rc == VINF_SUCCESS))
8872 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8873 else
8874 {
8875 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
8876 rc = VERR_EM_INTERPRETER;
8877 }
8878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
8879 return rc;
8880}
8881
8882
8883/**
8884 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
8885 */
8886HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8887{
8888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8889 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8890 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8891 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8892 AssertRCReturn(rc, rc);
8893
8894 PVM pVM = pVCpu->CTX_SUFF(pVM);
8895 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8896 rc = VBOXSTRICTRC_VAL(rc2);
8897 if (RT_LIKELY( rc == VINF_SUCCESS
8898 || rc == VINF_EM_HALT))
8899 {
8900 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8901 AssertRCReturn(rc3, rc3);
8902
8903 if ( rc == VINF_EM_HALT
8904 && EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
8905 {
8906 rc = VINF_SUCCESS;
8907 }
8908 }
8909 else
8910 {
8911 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
8912 rc = VERR_EM_INTERPRETER;
8913 }
8914 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
8915 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
8916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
8917 return rc;
8918}
8919
8920
8921/**
8922 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
8923 */
8924HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8925{
8926 /*
8927 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
8928 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
8929 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
8930 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
8931 */
8932 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8933 pVCpu->hm.s.u32HMError = VMX_EXIT_RSM;
8934 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8935}
8936
8937
8938/**
8939 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
8940 */
8941HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8942{
8943 /*
8944 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
8945 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
8946 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
8947 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
8948 */
8949 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8950 pVCpu->hm.s.u32HMError = VMX_EXIT_SMI;
8951 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8952}
8953
8954
8955/**
8956 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
8957 */
8958HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8959{
8960 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
8961 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8962 pVCpu->hm.s.u32HMError = VMX_EXIT_IO_SMI;
8963 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8964}
8965
8966
8967/**
8968 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
8969 */
8970HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8971{
8972 /*
8973 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
8974 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
8975 * See Intel spec. 25.3 "Other Causes of VM-exits".
8976 */
8977 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8978 pVCpu->hm.s.u32HMError = VMX_EXIT_SIPI;
8979 return VERR_VMX_UNEXPECTED_EXIT_CODE;
8980}
8981
8982
8983/**
8984 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
8985 * VM-exit.
8986 */
8987HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8988{
8989 /*
8990 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
8991 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
8992 *
8993 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
8994 * See Intel spec. "23.8 Restrictions on VMX operation".
8995 */
8996 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
8997 return VINF_SUCCESS;
8998}
8999
9000
9001/**
9002 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
9003 * VM-exit.
9004 */
9005HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9006{
9007 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9008 return VINF_EM_RESET;
9009}
9010
9011
9012/**
9013 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
9014 */
9015HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9016{
9017 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9018 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
9019 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9020 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9021 AssertRCReturn(rc, rc);
9022
9023 pMixedCtx->rip++;
9024 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9025 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
9026 rc = VINF_SUCCESS;
9027 else
9028 rc = VINF_EM_HALT;
9029
9030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
9031 return rc;
9032}
9033
9034
9035/**
9036 * VM-exit handler for instructions that result in a #UD exception delivered to
9037 * the guest.
9038 */
9039HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9040{
9041 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9042 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
9043 return VINF_SUCCESS;
9044}
9045
9046
9047/**
9048 * VM-exit handler for expiry of the VMX preemption timer.
9049 */
9050HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9051{
9052 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9053
9054 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
9055 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9056
9057 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
9058 PVM pVM = pVCpu->CTX_SUFF(pVM);
9059 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
9060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
9061 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
9062}
9063
9064
9065/**
9066 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
9067 */
9068HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9069{
9070 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9071
9072 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
9073 /** @todo check if XSETBV is supported by the recompiler. */
9074 return VERR_EM_INTERPRETER;
9075}
9076
9077
9078/**
9079 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
9080 */
9081HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9082{
9083 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9084
9085 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
9086 /** @todo implement EMInterpretInvpcid() */
9087 return VERR_EM_INTERPRETER;
9088}
9089
9090
9091/**
9092 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
9093 * Error VM-exit.
9094 */
9095HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9096{
9097 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9098 AssertRCReturn(rc, rc);
9099
9100 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
9101 NOREF(uInvalidReason);
9102
9103#ifdef VBOX_STRICT
9104 uint32_t uIntrState;
9105 HMVMXHCUINTREG uHCReg;
9106 uint64_t u64Val;
9107 uint32_t u32Val;
9108
9109 rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
9110 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
9111 rc |= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
9112 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
9113 AssertRCReturn(rc, rc);
9114
9115 Log4(("uInvalidReason %u\n", uInvalidReason));
9116 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
9117 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
9118 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
9119 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
9120
9121 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
9122 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
9123 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
9124 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
9125 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
9126 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9127 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
9128 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
9129 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
9130 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
9131 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
9132 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
9133#endif
9134
9135 PVM pVM = pVCpu->CTX_SUFF(pVM);
9136 HMDumpRegs(pVM, pVCpu, pMixedCtx);
9137
9138 return VERR_VMX_INVALID_GUEST_STATE;
9139}
9140
9141
9142/**
9143 * VM-exit handler for VM-entry failure due to an MSR-load
9144 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
9145 */
9146HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9147{
9148 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9149 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9150}
9151
9152
9153/**
9154 * VM-exit handler for VM-entry failure due to a machine-check event
9155 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
9156 */
9157HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9158{
9159 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9160 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9161}
9162
9163
9164/**
9165 * VM-exit handler for all undefined reasons. Should never ever happen.. in
9166 * theory.
9167 */
9168HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9169{
9170 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
9171 return VERR_VMX_UNDEFINED_EXIT_CODE;
9172}
9173
9174
9175/**
9176 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
9177 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
9178 * Conditional VM-exit.
9179 */
9180HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9181{
9182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9183
9184 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
9185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
9186 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
9187 return VERR_EM_INTERPRETER;
9188 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9189 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9190}
9191
9192
9193/**
9194 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
9195 */
9196HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9197{
9198 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9199
9200 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
9201 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
9202 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
9203 return VERR_EM_INTERPRETER;
9204 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9205 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9206}
9207
9208
9209/**
9210 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
9211 */
9212HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9213{
9214 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9215
9216 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
9217 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9218 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9219 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9220 AssertRCReturn(rc, rc);
9221 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
9222
9223 PVM pVM = pVCpu->CTX_SUFF(pVM);
9224 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9225 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
9226 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
9227 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
9228
9229 if (RT_LIKELY(rc == VINF_SUCCESS))
9230 {
9231 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9232 Assert(pVmxTransient->cbInstr == 2);
9233 }
9234 return rc;
9235}
9236
9237
9238/**
9239 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
9240 */
9241HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9242{
9243 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9244 PVM pVM = pVCpu->CTX_SUFF(pVM);
9245 int rc = VINF_SUCCESS;
9246
9247 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
9248 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9249 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9250 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9251 AssertRCReturn(rc, rc);
9252 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
9253
9254 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9255 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
9256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
9257
9258 if (RT_LIKELY(rc == VINF_SUCCESS))
9259 {
9260 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9261
9262 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
9263 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
9264 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
9265 {
9266 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
9267 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
9268 EMInterpretWrmsr() changes it. */
9269 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9270 }
9271 else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
9272 {
9273 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9274 AssertRCReturn(rc, rc);
9275 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
9276 }
9277 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
9278 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9279
9280 /* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
9281 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
9282 {
9283 switch (pMixedCtx->ecx)
9284 {
9285 case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
9286 case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
9287 case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
9288 case MSR_K8_FS_BASE: /* no break */
9289 case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS; break;
9290 case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
9291 }
9292 }
9293#ifdef VBOX_STRICT
9294 else
9295 {
9296 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
9297 switch (pMixedCtx->ecx)
9298 {
9299 case MSR_IA32_SYSENTER_CS:
9300 case MSR_IA32_SYSENTER_EIP:
9301 case MSR_IA32_SYSENTER_ESP:
9302 case MSR_K8_FS_BASE:
9303 case MSR_K8_GS_BASE:
9304 {
9305 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
9306 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9307 }
9308
9309 case MSR_K8_LSTAR:
9310 case MSR_K6_STAR:
9311 case MSR_K8_SF_MASK:
9312 case MSR_K8_TSC_AUX:
9313 case MSR_K8_KERNEL_GS_BASE:
9314 {
9315 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
9316 pMixedCtx->ecx));
9317 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9318 }
9319 }
9320 }
9321#endif /* VBOX_STRICT */
9322 }
9323 return rc;
9324}
9325
9326
9327/**
9328 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
9329 */
9330HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9331{
9332 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9333
9334 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
9335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
9336 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
9337 return VERR_EM_INTERPRETER;
9338 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9339 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9340}
9341
9342
9343/**
9344 * VM-exit handler for when the TPR value is lowered below the specified
9345 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
9346 */
9347HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9348{
9349 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9350 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
9351
9352 /*
9353 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
9354 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
9355 * resume guest execution.
9356 */
9357 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
9359 return VINF_SUCCESS;
9360}
9361
9362
9363/**
9364 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
9365 * VM-exit.
9366 *
9367 * @retval VINF_SUCCESS when guest execution can continue.
9368 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
9369 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
9370 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
9371 * recompiler.
9372 */
9373HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9374{
9375 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9376 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
9377 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9378 AssertRCReturn(rc, rc);
9379
9380 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
9381 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
9382 PVM pVM = pVCpu->CTX_SUFF(pVM);
9383 switch (uAccessType)
9384 {
9385 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
9386 {
9387#if 0
9388 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
9389 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9390#else
9391 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9392 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9393 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9394#endif
9395 AssertRCReturn(rc, rc);
9396
9397 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9398 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
9399 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
9400 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
9401
9402 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
9403 {
9404 case 0: /* CR0 */
9405 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
9406 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9407 break;
9408 case 2: /* C2 **/
9409 /* Nothing to do here, CR2 it's not part of the VMCS. */
9410 break;
9411 case 3: /* CR3 */
9412 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
9413 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
9414 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
9415 break;
9416 case 4: /* CR4 */
9417 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
9418 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
9419 break;
9420 case 8: /* CR8 */
9421 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9422 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
9423 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_VMX_GUEST_APIC_STATE;
9424 break;
9425 default:
9426 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
9427 break;
9428 }
9429
9430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9431 break;
9432 }
9433
9434 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
9435 {
9436 /* EMInterpretCRxRead() requires EFER MSR, CS. */
9437 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9438 AssertRCReturn(rc, rc);
9439 Assert( !pVM->hm.s.fNestedPaging
9440 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
9441 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
9442
9443 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
9444 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
9445 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
9446
9447 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9448 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
9449 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
9450 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9451 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
9452 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
9453 break;
9454 }
9455
9456 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
9457 {
9458 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9459 AssertRCReturn(rc, rc);
9460 rc = EMInterpretCLTS(pVM, pVCpu);
9461 AssertRCReturn(rc, rc);
9462 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
9464 Log4(("CRX CLTS write rc=%d\n", rc));
9465 break;
9466 }
9467
9468 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
9469 {
9470 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9471 AssertRCReturn(rc, rc);
9472 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
9473 if (RT_LIKELY(rc == VINF_SUCCESS))
9474 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
9475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
9476 Log4(("CRX LMSW write rc=%d\n", rc));
9477 break;
9478 }
9479
9480 default:
9481 {
9482 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
9483 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9484 }
9485 }
9486
9487 /* Validate possible error codes. */
9488 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
9489 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
9490 if (RT_SUCCESS(rc))
9491 {
9492 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9493 AssertRCReturn(rc2, rc2);
9494 }
9495
9496 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
9497 return rc;
9498}
9499
9500
9501/**
9502 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
9503 * VM-exit.
9504 */
9505HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9506{
9507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9508 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
9509
9510 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9511 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
9512 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9513 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
9514 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
9515 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
9516 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
9517 AssertRCReturn(rc2, rc2);
9518
9519 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
9520 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
9521 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
9522 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
9523 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
9524 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
9525 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
9526
9527 /* I/O operation lookup arrays. */
9528 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
9529 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
9530
9531 VBOXSTRICTRC rcStrict;
9532 const uint32_t cbValue = s_aIOSizes[uIOWidth];
9533 const uint32_t cbInstr = pVmxTransient->cbInstr;
9534 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
9535 PVM pVM = pVCpu->CTX_SUFF(pVM);
9536 if (fIOString)
9537 {
9538 /*
9539 * INS/OUTS - I/O String instruction.
9540 *
9541 * Use instruction-information if available, otherwise fall back on
9542 * interpreting the instruction.
9543 */
9544 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9545#if 0 /* Not quite ready, seem iSegReg assertion trigger once... Do we perhaps need to always read that in longjmp / preempt scenario? */
9546 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
9547 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
9548 {
9549 rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
9550 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9551 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9552 AssertRCReturn(rc2, rc2);
9553 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
9554 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
9555 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
9556 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
9557 if (fIOWrite)
9558 {
9559 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
9560 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9561 //if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9562 // hmR0SavePendingIOPortWriteStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr,
9563 // pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
9564 }
9565 else
9566 {
9567 AssertMsgReturn(pVmxTransient->ExitInstrInfo.StrIo.iSegReg == X86_SREG_ES,
9568 ("%#x (%#llx)\n", pVmxTransient->ExitInstrInfo.StrIo.iSegReg, pVmxTransient->ExitInstrInfo.u),
9569 VERR_HMVMX_IPE_4);
9570 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
9571 //if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9572 // hmR0SavePendingIOPortReadStr(pVCpu, pMixedCtx->rip, cbValue, enmAddrMode, fRep, cbInstr);
9573 }
9574 }
9575 else
9576 {
9577 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
9578 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9579 AssertRCReturn(rc2, rc2);
9580 rcStrict = IEMExecOne(pVCpu);
9581 }
9582 /** @todo IEM needs to be setting these flags somehow. */
9583 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9584 fUpdateRipAlready = true;
9585#else
9586 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
9587 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
9588 if (RT_SUCCESS(rcStrict))
9589 {
9590 if (fIOWrite)
9591 {
9592 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9593 (DISCPUMODE)pDis->uAddrMode, cbValue);
9594 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
9595 }
9596 else
9597 {
9598 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
9599 (DISCPUMODE)pDis->uAddrMode, cbValue);
9600 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
9601 }
9602 }
9603 else
9604 {
9605 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
9606 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
9607 }
9608#endif
9609 }
9610 else
9611 {
9612 /*
9613 * IN/OUT - I/O instruction.
9614 */
9615 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
9616 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
9617 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
9618 if (fIOWrite)
9619 {
9620 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
9621 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9622 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9623 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
9624 }
9625 else
9626 {
9627 uint32_t u32Result = 0;
9628 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
9629 if (IOM_SUCCESS(rcStrict))
9630 {
9631 /* Save result of I/O IN instr. in AL/AX/EAX. */
9632 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
9633 }
9634 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9635 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
9636 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
9637 }
9638 }
9639
9640 if (IOM_SUCCESS(rcStrict))
9641 {
9642 if (!fUpdateRipAlready)
9643 {
9644 pMixedCtx->rip += cbInstr;
9645 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
9646 }
9647
9648 /*
9649 * If any I/O breakpoints are armed, we need to check if one triggered
9650 * and take appropriate action.
9651 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
9652 */
9653 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
9654 AssertRCReturn(rc2, rc2);
9655
9656 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
9657 * execution engines about whether hyper BPs and such are pending. */
9658 uint32_t const uDr7 = pMixedCtx->dr[7];
9659 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
9660 && X86_DR7_ANY_RW_IO(uDr7)
9661 && (pMixedCtx->cr4 & X86_CR4_DE))
9662 || DBGFBpIsHwIoArmed(pVM)))
9663 {
9664 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
9665
9666 /* We're playing with the host CPU state here, make sure we don't preempt. */
9667 HM_DISABLE_PREEMPT_IF_NEEDED();
9668 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
9669
9670 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
9671 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
9672 {
9673 /* Raise #DB. */
9674 if (fIsGuestDbgActive)
9675 ASMSetDR6(pMixedCtx->dr[6]);
9676 if (pMixedCtx->dr[7] != uDr7)
9677 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9678
9679 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
9680 }
9681 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
9682 else if ( rcStrict2 != VINF_SUCCESS
9683 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
9684 rcStrict = rcStrict2;
9685
9686 HM_RESTORE_PREEMPT_IF_NEEDED();
9687 }
9688 }
9689
9690#ifdef DEBUG
9691 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
9692 Assert(!fIOWrite);
9693 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
9694 Assert(fIOWrite);
9695 else
9696 {
9697 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
9698 * statuses, that the VMM device and some others may return. See
9699 * IOM_SUCCESS() for guidance. */
9700 AssertMsg( RT_FAILURE(rcStrict)
9701 || rcStrict == VINF_SUCCESS
9702 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
9703 || rcStrict == VINF_EM_DBG_BREAKPOINT
9704 || rcStrict == VINF_EM_RAW_GUEST_TRAP
9705 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
9706 }
9707#endif
9708
9709 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
9710 return VBOXSTRICTRC_TODO(rcStrict);
9711}
9712
9713
9714/**
9715 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
9716 * VM-exit.
9717 */
9718HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9719{
9720 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9721
9722 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
9723 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9724 AssertRCReturn(rc, rc);
9725 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
9726 {
9727 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
9728 AssertRCReturn(rc, rc);
9729 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
9730 {
9731 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
9732
9733 /* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
9734 if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
9735 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
9736 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
9737 {
9738 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
9739 bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
9740
9741 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
9742 Assert(!pVCpu->hm.s.Event.fPending);
9743 pVCpu->hm.s.Event.fPending = true;
9744 pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
9745 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
9746 AssertRCReturn(rc, rc);
9747 if (fErrorCodeValid)
9748 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
9749 else
9750 pVCpu->hm.s.Event.u32ErrCode = 0;
9751 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
9752 && uVector == X86_XCPT_PF)
9753 {
9754 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
9755 }
9756
9757 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
9758 }
9759 }
9760 }
9761
9762 /** @todo Emulate task switch someday, currently just going back to ring-3 for
9763 * emulation. */
9764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
9765 return VERR_EM_INTERPRETER;
9766}
9767
9768
9769/**
9770 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
9771 */
9772HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9773{
9774 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9775 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
9776 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
9777 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9778 AssertRCReturn(rc, rc);
9779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
9780 return VINF_EM_DBG_STEPPED;
9781}
9782
9783
9784/**
9785 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
9786 */
9787HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788{
9789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9790
9791 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9792 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9793 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9794 return VINF_SUCCESS;
9795 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9796 return rc;
9797
9798#if 0
9799 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
9800 * just sync the whole thing. */
9801 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9802#else
9803 /* Aggressive state sync. for now. */
9804 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9805 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9806 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9807#endif
9808 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9809 AssertRCReturn(rc, rc);
9810
9811 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
9812 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
9813 switch (uAccessType)
9814 {
9815 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
9816 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
9817 {
9818 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9819 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
9820 {
9821 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
9822 }
9823
9824 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
9825 GCPhys &= PAGE_BASE_GC_MASK;
9826 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
9827 PVM pVM = pVCpu->CTX_SUFF(pVM);
9828 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
9829 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
9830
9831 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
9832 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
9833 CPUMCTX2CORE(pMixedCtx), GCPhys);
9834 rc = VBOXSTRICTRC_VAL(rc2);
9835 Log4(("ApicAccess rc=%d\n", rc));
9836 if ( rc == VINF_SUCCESS
9837 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9838 || rc == VERR_PAGE_NOT_PRESENT)
9839 {
9840 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9841 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9842 rc = VINF_SUCCESS;
9843 }
9844 break;
9845 }
9846
9847 default:
9848 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
9849 rc = VINF_EM_RAW_EMULATE_INSTR;
9850 break;
9851 }
9852
9853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
9854 return rc;
9855}
9856
9857
9858/**
9859 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
9860 * VM-exit.
9861 */
9862HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9863{
9864 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9865
9866 /* We should -not- get this VM-exit if the guest is debugging. */
9867 if (CPUMIsGuestDebugStateActive(pVCpu))
9868 {
9869 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9870 return VERR_VMX_UNEXPECTED_EXIT_CODE;
9871 }
9872
9873 int rc = VERR_INTERNAL_ERROR_5;
9874 if ( !DBGFIsStepping(pVCpu)
9875 && !pVCpu->hm.s.fSingleInstruction
9876 && !CPUMIsHyperDebugStateActive(pVCpu))
9877 {
9878 /* Don't intercept MOV DRx and #DB any more. */
9879 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
9880 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9881 AssertRCReturn(rc, rc);
9882
9883 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9884 {
9885#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9886 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
9887 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9888 AssertRCReturn(rc, rc);
9889#endif
9890 }
9891
9892 /* We're playing with the host CPU state here, make sure we can't preempt. */
9893 HM_DISABLE_PREEMPT_IF_NEEDED();
9894
9895 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
9896 PVM pVM = pVCpu->CTX_SUFF(pVM);
9897 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
9898 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
9899
9900 HM_RESTORE_PREEMPT_IF_NEEDED();
9901
9902#ifdef VBOX_WITH_STATISTICS
9903 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9904 AssertRCReturn(rc, rc);
9905 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9906 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9907 else
9908 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9909#endif
9910 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
9911 return VINF_SUCCESS;
9912 }
9913
9914 /*
9915 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
9916 * hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
9917 */
9918 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9919 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9920 AssertRCReturn(rc, rc);
9921 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
9922
9923 PVM pVM = pVCpu->CTX_SUFF(pVM);
9924 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
9925 {
9926 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9927 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
9928 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
9929 if (RT_SUCCESS(rc))
9930 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
9931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
9932 }
9933 else
9934 {
9935 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
9936 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
9937 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
9938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
9939 }
9940
9941 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
9942 if (RT_SUCCESS(rc))
9943 {
9944 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9945 AssertRCReturn(rc2, rc2);
9946 }
9947 return rc;
9948}
9949
9950
9951/**
9952 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
9953 * Conditional VM-exit.
9954 */
9955HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9956{
9957 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9958 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
9959
9960 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9961 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9962 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9963 return VINF_SUCCESS;
9964 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9965 return rc;
9966
9967 RTGCPHYS GCPhys = 0;
9968 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
9969
9970#if 0
9971 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
9972#else
9973 /* Aggressive state sync. for now. */
9974 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
9975 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9976 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9977#endif
9978 AssertRCReturn(rc, rc);
9979
9980 /*
9981 * If we succeed, resume guest execution.
9982 * If we fail in interpreting the instruction because we couldn't get the guest physical address
9983 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
9984 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
9985 * weird case. See @bugref{6043}.
9986 */
9987 PVM pVM = pVCpu->CTX_SUFF(pVM);
9988 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
9989 rc = VBOXSTRICTRC_VAL(rc2);
9990 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
9991 if ( rc == VINF_SUCCESS
9992 || rc == VERR_PAGE_TABLE_NOT_PRESENT
9993 || rc == VERR_PAGE_NOT_PRESENT)
9994 {
9995 /* Successfully handled MMIO operation. */
9996 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
9997 | HM_CHANGED_VMX_GUEST_APIC_STATE;
9998 rc = VINF_SUCCESS;
9999 }
10000 return rc;
10001}
10002
10003
10004/**
10005 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
10006 * VM-exit.
10007 */
10008HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10009{
10010 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10011 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
10012
10013 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10014 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10015 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10016 return VINF_SUCCESS;
10017 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10018 return rc;
10019
10020 RTGCPHYS GCPhys = 0;
10021 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
10022 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10023#if 0
10024 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
10025#else
10026 /* Aggressive state sync. for now. */
10027 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10028 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10029 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10030#endif
10031 AssertRCReturn(rc, rc);
10032
10033 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
10034 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
10035
10036 RTGCUINT uErrorCode = 0;
10037 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
10038 uErrorCode |= X86_TRAP_PF_ID;
10039 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
10040 uErrorCode |= X86_TRAP_PF_RW;
10041 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
10042 uErrorCode |= X86_TRAP_PF_P;
10043
10044 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
10045
10046 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
10047 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10048
10049 /* Handle the pagefault trap for the nested shadow table. */
10050 PVM pVM = pVCpu->CTX_SUFF(pVM);
10051 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
10052 TRPMResetTrap(pVCpu);
10053
10054 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
10055 if ( rc == VINF_SUCCESS
10056 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10057 || rc == VERR_PAGE_NOT_PRESENT)
10058 {
10059 /* Successfully synced our nested page tables. */
10060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
10061 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10062 return VINF_SUCCESS;
10063 }
10064
10065 Log4(("EPT return to ring-3 rc=%d\n"));
10066 return rc;
10067}
10068
10069/** @} */
10070
10071/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10072/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
10073/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10074
10075/** @name VM-exit exception handlers.
10076 * @{
10077 */
10078
10079/**
10080 * VM-exit exception handler for #MF (Math Fault: floating point exception).
10081 */
10082static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10083{
10084 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
10086
10087 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10088 AssertRCReturn(rc, rc);
10089
10090 if (!(pMixedCtx->cr0 & X86_CR0_NE))
10091 {
10092 /* Old-style FPU error reporting needs some extra work. */
10093 /** @todo don't fall back to the recompiler, but do it manually. */
10094 return VERR_EM_INTERPRETER;
10095 }
10096
10097 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10098 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10099 return rc;
10100}
10101
10102
10103/**
10104 * VM-exit exception handler for #BP (Breakpoint exception).
10105 */
10106static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10107{
10108 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
10110
10111 /** @todo Try optimize this by not saving the entire guest state unless
10112 * really needed. */
10113 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10114 AssertRCReturn(rc, rc);
10115
10116 PVM pVM = pVCpu->CTX_SUFF(pVM);
10117 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10118 if (rc == VINF_EM_RAW_GUEST_TRAP)
10119 {
10120 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10121 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10122 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10123 AssertRCReturn(rc, rc);
10124
10125 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10126 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10127 }
10128
10129 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
10130 return rc;
10131}
10132
10133
10134/**
10135 * VM-exit exception handler for #DB (Debug exception).
10136 */
10137static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10138{
10139 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
10141 Log6(("XcptDB\n"));
10142
10143 /*
10144 * Get the DR6-like values from the exit qualification and pass it to DBGF
10145 * for processing.
10146 */
10147 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10148 AssertRCReturn(rc, rc);
10149
10150 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
10151 uint64_t uDR6 = X86_DR6_INIT_VAL;
10152 uDR6 |= ( pVmxTransient->uExitQualification
10153 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
10154
10155 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
10156 if (rc == VINF_EM_RAW_GUEST_TRAP)
10157 {
10158 /*
10159 * The exception was for the guest. Update DR6, DR7.GD and
10160 * IA32_DEBUGCTL.LBR before forwarding it.
10161 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
10162 */
10163 HM_DISABLE_PREEMPT_IF_NEEDED();
10164
10165 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
10166 pMixedCtx->dr[6] |= uDR6;
10167 if (CPUMIsGuestDebugStateActive(pVCpu))
10168 ASMSetDR6(pMixedCtx->dr[6]);
10169
10170 HM_RESTORE_PREEMPT_IF_NEEDED();
10171
10172 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10173 AssertRCReturn(rc, rc);
10174
10175 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
10176 pMixedCtx->dr[7] &= ~X86_DR7_GD;
10177
10178 /* Paranoia. */
10179 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
10180 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
10181
10182 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
10183 AssertRCReturn(rc, rc);
10184
10185 /*
10186 * Raise #DB in the guest.
10187 */
10188 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10189 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10190 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10191 AssertRCReturn(rc, rc);
10192 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10193 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10194 return VINF_SUCCESS;
10195 }
10196
10197 /*
10198 * Not a guest trap, must be a hypervisor related debug event then.
10199 * Update DR6 in case someone is interested in it.
10200 */
10201 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
10202 AssertReturn(CPUMIsHyperDebugStateActive(pVCpu), VERR_HM_IPE_5);
10203 CPUMSetHyperDR6(pVCpu, uDR6);
10204
10205 return rc;
10206}
10207
10208
10209/**
10210 * VM-exit exception handler for #NM (Device-not-available exception: floating
10211 * point exception).
10212 */
10213static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10214{
10215 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10216
10217#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10218 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
10219#endif
10220
10221 /* We require CR0 and EFER. EFER is always up-to-date. */
10222 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10223 AssertRCReturn(rc, rc);
10224
10225 /* We're playing with the host CPU state here, have to disable preemption. */
10226 HM_DISABLE_PREEMPT_IF_NEEDED();
10227
10228 /* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
10229 PVM pVM = pVCpu->CTX_SUFF(pVM);
10230 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
10231 if (rc == VINF_SUCCESS)
10232 {
10233 Assert(CPUMIsGuestFPUStateActive(pVCpu));
10234 HM_RESTORE_PREEMPT_IF_NEEDED();
10235
10236 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
10237 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
10238 return VINF_SUCCESS;
10239 }
10240
10241 HM_RESTORE_PREEMPT_IF_NEEDED();
10242
10243 /* Forward #NM to the guest. */
10244 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
10245 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10246 AssertRCReturn(rc, rc);
10247 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10248 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
10249 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10250 return rc;
10251}
10252
10253
10254/**
10255 * VM-exit exception handler for #GP (General-protection exception).
10256 *
10257 * @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
10258 */
10259static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10260{
10261 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
10263
10264 int rc = VERR_INTERNAL_ERROR_5;
10265 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10266 {
10267#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
10268 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
10269 rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10270 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10271 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10272 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10273 AssertRCReturn(rc, rc);
10274 Log4(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
10275 pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
10276 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10277 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10278 return rc;
10279#else
10280 /* We don't intercept #GP. */
10281 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
10282 return VERR_VMX_UNEXPECTED_EXCEPTION;
10283#endif
10284 }
10285
10286 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
10287 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
10288
10289 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
10290 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10291 AssertRCReturn(rc, rc);
10292
10293 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10294 uint32_t cbOp = 0;
10295 PVM pVM = pVCpu->CTX_SUFF(pVM);
10296 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
10297 if (RT_SUCCESS(rc))
10298 {
10299 rc = VINF_SUCCESS;
10300 Assert(cbOp == pDis->cbInstr);
10301 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
10302 switch (pDis->pCurInstr->uOpcode)
10303 {
10304 case OP_CLI:
10305 {
10306 pMixedCtx->eflags.Bits.u1IF = 0;
10307 pMixedCtx->rip += pDis->cbInstr;
10308 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10309 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
10310 break;
10311 }
10312
10313 case OP_STI:
10314 {
10315 pMixedCtx->eflags.Bits.u1IF = 1;
10316 pMixedCtx->rip += pDis->cbInstr;
10317 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
10318 Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
10319 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS;
10320 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
10321 break;
10322 }
10323
10324 case OP_HLT:
10325 {
10326 rc = VINF_EM_HALT;
10327 pMixedCtx->rip += pDis->cbInstr;
10328 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP;
10329 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10330 break;
10331 }
10332
10333 case OP_POPF:
10334 {
10335 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
10336 uint32_t cbParm = 0;
10337 uint32_t uMask = 0;
10338 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10339 {
10340 cbParm = 4;
10341 uMask = 0xffffffff;
10342 }
10343 else
10344 {
10345 cbParm = 2;
10346 uMask = 0xffff;
10347 }
10348
10349 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
10350 RTGCPTR GCPtrStack = 0;
10351 X86EFLAGS Eflags;
10352 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10353 &GCPtrStack);
10354 if (RT_SUCCESS(rc))
10355 {
10356 Assert(sizeof(Eflags.u32) >= cbParm);
10357 Eflags.u32 = 0;
10358 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
10359 }
10360 if (RT_FAILURE(rc))
10361 {
10362 rc = VERR_EM_INTERPRETER;
10363 break;
10364 }
10365 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
10366 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10367 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
10368 /* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
10369 pMixedCtx->eflags.Bits.u1RF = 0;
10370 pMixedCtx->esp += cbParm;
10371 pMixedCtx->esp &= uMask;
10372 pMixedCtx->rip += pDis->cbInstr;
10373 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS;
10374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
10375 break;
10376 }
10377
10378 case OP_PUSHF:
10379 {
10380 uint32_t cbParm = 0;
10381 uint32_t uMask = 0;
10382 if (pDis->fPrefix & DISPREFIX_OPSIZE)
10383 {
10384 cbParm = 4;
10385 uMask = 0xffffffff;
10386 }
10387 else
10388 {
10389 cbParm = 2;
10390 uMask = 0xffff;
10391 }
10392
10393 /* Get the stack pointer & push the contents of eflags onto the stack. */
10394 RTGCPTR GCPtrStack = 0;
10395 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
10396 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
10397 if (RT_FAILURE(rc))
10398 {
10399 rc = VERR_EM_INTERPRETER;
10400 break;
10401 }
10402 X86EFLAGS Eflags = pMixedCtx->eflags;
10403 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
10404 Eflags.Bits.u1RF = 0;
10405 Eflags.Bits.u1VM = 0;
10406
10407 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
10408 if (RT_FAILURE(rc))
10409 {
10410 rc = VERR_EM_INTERPRETER;
10411 break;
10412 }
10413 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
10414 pMixedCtx->esp -= cbParm;
10415 pMixedCtx->esp &= uMask;
10416 pMixedCtx->rip += pDis->cbInstr;
10417 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP;
10418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
10419 break;
10420 }
10421
10422 case OP_IRET:
10423 {
10424 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
10425 * instruction reference. */
10426 RTGCPTR GCPtrStack = 0;
10427 uint32_t uMask = 0xffff;
10428 uint16_t aIretFrame[3];
10429 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
10430 {
10431 rc = VERR_EM_INTERPRETER;
10432 break;
10433 }
10434 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
10435 &GCPtrStack);
10436 if (RT_SUCCESS(rc))
10437 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
10438 if (RT_FAILURE(rc))
10439 {
10440 rc = VERR_EM_INTERPRETER;
10441 break;
10442 }
10443 pMixedCtx->eip = 0;
10444 pMixedCtx->ip = aIretFrame[0];
10445 pMixedCtx->cs.Sel = aIretFrame[1];
10446 pMixedCtx->cs.ValidSel = aIretFrame[1];
10447 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
10448 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
10449 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
10450 pMixedCtx->sp += sizeof(aIretFrame);
10451 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_SEGMENT_REGS | HM_CHANGED_GUEST_RSP
10452 | HM_CHANGED_GUEST_RFLAGS;
10453 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
10454 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
10455 break;
10456 }
10457
10458 case OP_INT:
10459 {
10460 uint16_t uVector = pDis->Param1.uValue & 0xff;
10461 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
10462 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10463 break;
10464 }
10465
10466 case OP_INTO:
10467 {
10468 if (pMixedCtx->eflags.Bits.u1OF)
10469 {
10470 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
10471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
10472 }
10473 break;
10474 }
10475
10476 default:
10477 {
10478 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
10479 EMCODETYPE_SUPERVISOR);
10480 rc = VBOXSTRICTRC_VAL(rc2);
10481 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL_GUEST;
10482 Log4(("#GP rc=%Rrc\n", rc));
10483 break;
10484 }
10485 }
10486 }
10487 else
10488 rc = VERR_EM_INTERPRETER;
10489
10490 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
10491 ("#GP Unexpected rc=%Rrc\n", rc));
10492 return rc;
10493}
10494
10495
10496/**
10497 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
10498 * the exception reported in the VMX transient structure back into the VM.
10499 *
10500 * @remarks Requires uExitIntrInfo in the VMX transient structure to be
10501 * up-to-date.
10502 */
10503static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10504{
10505 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10506
10507 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
10508 hmR0VmxCheckExitDueToEventDelivery(). */
10509 int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10510 rc |= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
10511 AssertRCReturn(rc, rc);
10512 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
10513
10514 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10515 pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
10516 return VINF_SUCCESS;
10517}
10518
10519
10520/**
10521 * VM-exit exception handler for #PF (Page-fault exception).
10522 */
10523static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10524{
10525 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
10526 PVM pVM = pVCpu->CTX_SUFF(pVM);
10527 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10528 rc |= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
10529 rc |= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
10530 AssertRCReturn(rc, rc);
10531
10532#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
10533 if (pVM->hm.s.fNestedPaging)
10534 {
10535 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
10536 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
10537 {
10538 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10539 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10540 0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
10541 }
10542 else
10543 {
10544 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10545 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10546 Log4(("Pending #DF due to vectoring #PF. NP\n"));
10547 }
10548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10549 return rc;
10550 }
10551#else
10552 Assert(!pVM->hm.s.fNestedPaging);
10553#endif
10554
10555 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10556 AssertRCReturn(rc, rc);
10557
10558 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
10559 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
10560
10561 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
10562 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
10563 (RTGCPTR)pVmxTransient->uExitQualification);
10564
10565 Log4(("#PF: rc=%Rrc\n", rc));
10566 if (rc == VINF_SUCCESS)
10567 {
10568 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
10569 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
10570 * memory? We don't update the whole state here... */
10571 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
10572 | HM_CHANGED_VMX_GUEST_APIC_STATE;
10573 TRPMResetTrap(pVCpu);
10574 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
10575 return rc;
10576 }
10577 else if (rc == VINF_EM_RAW_GUEST_TRAP)
10578 {
10579 if (!pVmxTransient->fVectoringPF)
10580 {
10581 /* It's a guest page fault and needs to be reflected to the guest. */
10582 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
10583 TRPMResetTrap(pVCpu);
10584 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
10585 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
10586 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
10587 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
10588 }
10589 else
10590 {
10591 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
10592 TRPMResetTrap(pVCpu);
10593 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
10594 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
10595 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
10596 }
10597
10598 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
10599 return VINF_SUCCESS;
10600 }
10601
10602 TRPMResetTrap(pVCpu);
10603 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
10604 return rc;
10605}
10606
10607/** @} */
10608
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette