HMR0A.asm@ 87310

Last change on this file since 87310 was 87310, checked in by vboxsync, 4 years ago
VMM: Split out the generic VMX/SVM instruction wrapper utility method into a separate file as we'd probably like to keep this compiling on 32-bit too for testing purposes.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 41.3 KB

Line
1	; $Id: HMR0A.asm 87310 2021-01-19 23:39:19Z vboxsync $
2	;; @file
3	; HM - Ring-0 VMX, SVM world-switch and helper routines.
4	;
5
6	;
7	; Copyright (C) 2006-2020 Oracle Corporation
8	;
9	; This file is part of VirtualBox Open Source Edition (OSE), as
10	; available from http://www.215389.xyz. This file is free software;
11	; you can redistribute it and/or modify it under the terms of the GNU
12	; General Public License (GPL) as published by the Free Software
13	; Foundation, in version 2 as it comes in the "COPYING" file of the
14	; VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	; hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	;
17
18	;*********************************************************************************************************************************
19	;* Header Files *
20	;*********************************************************************************************************************************
21	%include "VBox/asmdefs.mac"
22	%include "VBox/err.mac"
23	%include "VBox/vmm/hm_vmx.mac"
24	%include "VBox/vmm/cpum.mac"
25	%include "VBox/vmm/vm.mac"
26	%include "iprt/x86.mac"
27	%include "HMInternal.mac"
28
29	%ifndef RT_ARCH_AMD64
30	%error AMD64 only.
31	%endif
32
33
34	;*********************************************************************************************************************************
35	;* Defined Constants And Macros *
36	;*********************************************************************************************************************************
37	;; The offset of the XMM registers in X86FXSTATE.
38	; Use define because I'm too lazy to convert the struct.
39	%define XMM_OFF_IN_X86FXSTATE 160
40
41	;; Spectre filler for 32-bit mode.
42	; Some user space address that points to a 4MB page boundrary in hope that it
43	; will somehow make it less useful.
44	%define SPECTRE_FILLER32 0x227fffff
45	;; Spectre filler for 64-bit mode.
46	; Choosen to be an invalid address (also with 5 level paging).
47	%define SPECTRE_FILLER64 0x02204204207fffff
48	;; Spectre filler for the current CPU mode.
49	%ifdef RT_ARCH_AMD64
50	%define SPECTRE_FILLER SPECTRE_FILLER64
51	%else
52	%define SPECTRE_FILLER SPECTRE_FILLER32
53	%endif
54
55	;;
56	; Determine skipping restoring of GDTR, IDTR, TR across VMX non-root operation.
57	;
58	%ifdef RT_ARCH_AMD64
59	%define VMX_SKIP_GDTR
60	%define VMX_SKIP_TR
61	%define VBOX_SKIP_RESTORE_SEG
62	%ifdef RT_OS_DARWIN
63	; Load the NULL selector into DS, ES, FS and GS on 64-bit darwin so we don't
64	; risk loading a stale LDT value or something invalid.
65	%define HM_64_BIT_USE_NULL_SEL
66	; Darwin (Mavericks) uses IDTR limit to store the CPU Id so we need to restore it always.
67	; See @bugref{6875}.
68	%else
69	%define VMX_SKIP_IDTR
70	%endif
71	%endif
72
73	;; @def MYPUSHAD
74	; Macro generating an equivalent to PUSHAD instruction.
75
76	;; @def MYPOPAD
77	; Macro generating an equivalent to POPAD instruction.
78
79	;; @def MYPUSHSEGS
80	; Macro saving all segment registers on the stack.
81	; @param 1 Full width register name.
82	; @param 2 16-bit register name for \a 1.
83
84	;; @def MYPOPSEGS
85	; Macro restoring all segment registers on the stack.
86	; @param 1 Full width register name.
87	; @param 2 16-bit register name for \a 1.
88
89	%ifdef ASM_CALL64_GCC
90	%macro MYPUSHAD64 0
91	push r15
92	push r14
93	push r13
94	push r12
95	push rbx
96	%endmacro
97	%macro MYPOPAD64 0
98	pop rbx
99	pop r12
100	pop r13
101	pop r14
102	pop r15
103	%endmacro
104
105	%else ; ASM_CALL64_MSC
106	%macro MYPUSHAD64 0
107	push r15
108	push r14
109	push r13
110	push r12
111	push rbx
112	push rsi
113	push rdi
114	%endmacro
115	%macro MYPOPAD64 0
116	pop rdi
117	pop rsi
118	pop rbx
119	pop r12
120	pop r13
121	pop r14
122	pop r15
123	%endmacro
124	%endif
125
126	%ifdef VBOX_SKIP_RESTORE_SEG
127	%macro MYPUSHSEGS64 2
128	%endmacro
129
130	%macro MYPOPSEGS64 2
131	%endmacro
132	%else ; !VBOX_SKIP_RESTORE_SEG
133	; Trashes, rax, rdx & rcx.
134	%macro MYPUSHSEGS64 2
135	%ifndef HM_64_BIT_USE_NULL_SEL
136	mov %2, es
137	push %1
138	mov %2, ds
139	push %1
140	%endif
141
142	; Special case for FS; Windows and Linux either don't use it or restore it when leaving kernel mode,
143	; Solaris OTOH doesn't and we must save it.
144	mov ecx, MSR_K8_FS_BASE
145	rdmsr
146	push rdx
147	push rax
148	%ifndef HM_64_BIT_USE_NULL_SEL
149	push fs
150	%endif
151
152	; Special case for GS; OSes typically use swapgs to reset the hidden base register for GS on entry into the kernel.
153	; The same happens on exit.
154	mov ecx, MSR_K8_GS_BASE
155	rdmsr
156	push rdx
157	push rax
158	%ifndef HM_64_BIT_USE_NULL_SEL
159	push gs
160	%endif
161	%endmacro
162
163	; trashes, rax, rdx & rcx
164	%macro MYPOPSEGS64 2
165	; Note: do not step through this code with a debugger!
166	%ifndef HM_64_BIT_USE_NULL_SEL
167	xor eax, eax
168	mov ds, ax
169	mov es, ax
170	mov fs, ax
171	mov gs, ax
172	%endif
173
174	%ifndef HM_64_BIT_USE_NULL_SEL
175	pop gs
176	%endif
177	pop rax
178	pop rdx
179	mov ecx, MSR_K8_GS_BASE
180	wrmsr
181
182	%ifndef HM_64_BIT_USE_NULL_SEL
183	pop fs
184	%endif
185	pop rax
186	pop rdx
187	mov ecx, MSR_K8_FS_BASE
188	wrmsr
189	; Now it's safe to step again
190
191	%ifndef HM_64_BIT_USE_NULL_SEL
192	pop %1
193	mov ds, %2
194	pop %1
195	mov es, %2
196	%endif
197	%endmacro
198	%endif ; VBOX_SKIP_RESTORE_SEG
199
200	%macro MYPUSHAD32 0
201	pushad
202	%endmacro
203	%macro MYPOPAD32 0
204	popad
205	%endmacro
206
207	%macro MYPUSHSEGS32 2
208	push ds
209	push es
210	push fs
211	push gs
212	%endmacro
213	%macro MYPOPSEGS32 2
214	pop gs
215	pop fs
216	pop es
217	pop ds
218	%endmacro
219
220	%ifdef RT_ARCH_AMD64
221	%define MYPUSHAD MYPUSHAD64
222	%define MYPOPAD MYPOPAD64
223	%define MYPUSHSEGS MYPUSHSEGS64
224	%define MYPOPSEGS MYPOPSEGS64
225	%else
226	%define MYPUSHAD MYPUSHAD32
227	%define MYPOPAD MYPOPAD32
228	%define MYPUSHSEGS MYPUSHSEGS32
229	%define MYPOPSEGS MYPOPSEGS32
230	%endif
231
232	;;
233	; Creates an indirect branch prediction barrier on CPUs that need and supports that.
234	; @clobbers eax, edx, ecx
235	; @param 1 How to address CPUMCTX.
236	; @param 2 Which flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
237	%macro INDIRECT_BRANCH_PREDICTION_BARRIER 2
238	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
239	jz %%no_indirect_branch_barrier
240	mov ecx, MSR_IA32_PRED_CMD
241	mov eax, MSR_IA32_PRED_CMD_F_IBPB
242	xor edx, edx
243	wrmsr
244	%%no_indirect_branch_barrier:
245	%endmacro
246
247	;;
248	; Creates an indirect branch prediction and L1D barrier on CPUs that need and supports that.
249	; @clobbers eax, edx, ecx
250	; @param 1 How to address CPUMCTX.
251	; @param 2 Which IBPB flag to test for (CPUMCTX_WSF_IBPB_ENTRY or CPUMCTX_WSF_IBPB_EXIT)
252	; @param 3 Which FLUSH flag to test for (CPUMCTX_WSF_L1D_ENTRY)
253	; @param 4 Which MDS flag to test for (CPUMCTX_WSF_MDS_ENTRY)
254	%macro INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER 4
255	; Only one test+jmp when disabled CPUs.
256	test byte [%1 + CPUMCTX.fWorldSwitcher], (%2 \| %3 \| %4)
257	jz %%no_barrier_needed
258
259	; The eax:edx value is the same for both.
260	AssertCompile(MSR_IA32_PRED_CMD_F_IBPB == MSR_IA32_FLUSH_CMD_F_L1D)
261	mov eax, MSR_IA32_PRED_CMD_F_IBPB
262	xor edx, edx
263
264	; Indirect branch barrier.
265	test byte [%1 + CPUMCTX.fWorldSwitcher], %2
266	jz %%no_indirect_branch_barrier
267	mov ecx, MSR_IA32_PRED_CMD
268	wrmsr
269	%%no_indirect_branch_barrier:
270
271	; Level 1 data cache flush.
272	test byte [%1 + CPUMCTX.fWorldSwitcher], %3
273	jz %%no_cache_flush_barrier
274	mov ecx, MSR_IA32_FLUSH_CMD
275	wrmsr
276	jmp %%no_mds_buffer_flushing ; MDS flushing is included in L1D_FLUSH
277	%%no_cache_flush_barrier:
278
279	; MDS buffer flushing.
280	test byte [%1 + CPUMCTX.fWorldSwitcher], %4
281	jz %%no_mds_buffer_flushing
282	sub xSP, xSP
283	mov [xSP], ds
284	verw [xSP]
285	add xSP, xSP
286	%%no_mds_buffer_flushing:
287
288	%%no_barrier_needed:
289	%endmacro
290
291
292	;*********************************************************************************************************************************
293	;* External Symbols *
294	;*********************************************************************************************************************************
295	%ifdef VBOX_WITH_KERNEL_USING_XMM
296	extern NAME(CPUMIsGuestFPUStateActive)
297	%endif
298
299
300	BEGINCODE
301
302
303	;;
304	; Restores host-state fields.
305	;
306	; @returns VBox status code
307	; @param f32RestoreHost x86: [ebp + 08h] msc: ecx gcc: edi RestoreHost flags.
308	; @param pRestoreHost x86: [ebp + 0ch] msc: rdx gcc: rsi Pointer to the RestoreHost struct.
309	;
310	ALIGNCODE(16)
311	BEGINPROC VMXRestoreHostState
312	%ifdef RT_ARCH_AMD64
313	%ifndef ASM_CALL64_GCC
314	; Use GCC's input registers since we'll be needing both rcx and rdx further
315	; down with the wrmsr instruction. Use the R10 and R11 register for saving
316	; RDI and RSI since MSC preserve the two latter registers.
317	mov r10, rdi
318	mov r11, rsi
319	mov rdi, rcx
320	mov rsi, rdx
321	%endif
322
323	test edi, VMX_RESTORE_HOST_GDTR
324	jz .test_idtr
325	lgdt [rsi + VMXRESTOREHOST.HostGdtr]
326
327	.test_idtr:
328	test edi, VMX_RESTORE_HOST_IDTR
329	jz .test_ds
330	lidt [rsi + VMXRESTOREHOST.HostIdtr]
331
332	.test_ds:
333	test edi, VMX_RESTORE_HOST_SEL_DS
334	jz .test_es
335	mov ax, [rsi + VMXRESTOREHOST.uHostSelDS]
336	mov ds, eax
337
338	.test_es:
339	test edi, VMX_RESTORE_HOST_SEL_ES
340	jz .test_tr
341	mov ax, [rsi + VMXRESTOREHOST.uHostSelES]
342	mov es, eax
343
344	.test_tr:
345	test edi, VMX_RESTORE_HOST_SEL_TR
346	jz .test_fs
347	; When restoring the TR, we must first clear the busy flag or we'll end up faulting.
348	mov dx, [rsi + VMXRESTOREHOST.uHostSelTR]
349	mov ax, dx
350	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
351	test edi, VMX_RESTORE_HOST_GDT_READ_ONLY \| VMX_RESTORE_HOST_GDT_NEED_WRITABLE
352	jnz .gdt_readonly
353	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
354	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
355	ltr dx
356	jmp short .test_fs
357	.gdt_readonly:
358	test edi, VMX_RESTORE_HOST_GDT_NEED_WRITABLE
359	jnz .gdt_readonly_need_writable
360	mov rcx, cr0
361	mov r9, rcx
362	add rax, qword [rsi + VMXRESTOREHOST.HostGdtr + 2] ; xAX <- descriptor offset + GDTR.pGdt.
363	and rcx, ~X86_CR0_WP
364	mov cr0, rcx
365	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
366	ltr dx
367	mov cr0, r9
368	jmp short .test_fs
369	.gdt_readonly_need_writable:
370	add rax, qword [rsi + VMXRESTOREHOST.HostGdtrRw + 2] ; xAX <- descriptor offset + GDTR.pGdtRw
371	and dword [rax + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
372	lgdt [rsi + VMXRESTOREHOST.HostGdtrRw]
373	ltr dx
374	lgdt [rsi + VMXRESTOREHOST.HostGdtr] ; load the original GDT
375
376	.test_fs:
377	;
378	; When restoring the selector values for FS and GS, we'll temporarily trash
379	; the base address (at least the high 32-bit bits, but quite possibly the
380	; whole base address), the wrmsr will restore it correctly. (VT-x actually
381	; restores the base correctly when leaving guest mode, but not the selector
382	; value, so there is little problem with interrupts being enabled prior to
383	; this restore job.)
384	; We'll disable ints once for both FS and GS as that's probably faster.
385	;
386	test edi, VMX_RESTORE_HOST_SEL_FS \| VMX_RESTORE_HOST_SEL_GS
387	jz .restore_success
388	pushfq
389	cli ; (see above)
390
391	test edi, VMX_RESTORE_HOST_SEL_FS
392	jz .test_gs
393	mov ax, word [rsi + VMXRESTOREHOST.uHostSelFS]
394	mov fs, eax
395	mov eax, dword [rsi + VMXRESTOREHOST.uHostFSBase] ; uHostFSBase - Lo
396	mov edx, dword [rsi + VMXRESTOREHOST.uHostFSBase + 4h] ; uHostFSBase - Hi
397	mov ecx, MSR_K8_FS_BASE
398	wrmsr
399
400	.test_gs:
401	test edi, VMX_RESTORE_HOST_SEL_GS
402	jz .restore_flags
403	mov ax, word [rsi + VMXRESTOREHOST.uHostSelGS]
404	mov gs, eax
405	mov eax, dword [rsi + VMXRESTOREHOST.uHostGSBase] ; uHostGSBase - Lo
406	mov edx, dword [rsi + VMXRESTOREHOST.uHostGSBase + 4h] ; uHostGSBase - Hi
407	mov ecx, MSR_K8_GS_BASE
408	wrmsr
409
410	.restore_flags:
411	popfq
412
413	.restore_success:
414	mov eax, VINF_SUCCESS
415	%ifndef ASM_CALL64_GCC
416	; Restore RDI and RSI on MSC.
417	mov rdi, r10
418	mov rsi, r11
419	%endif
420	%else ; RT_ARCH_X86
421	mov eax, VERR_NOT_IMPLEMENTED
422	%endif
423	ret
424	ENDPROC VMXRestoreHostState
425
426
427	;;
428	; Dispatches an NMI to the host.
429	;
430	ALIGNCODE(16)
431	BEGINPROC VMXDispatchHostNmi
432	; NMI is always vector 2. The IDT[2] IRQ handler cannot be anything else. See Intel spec. 6.3.1 "External Interrupts".
433	int 2
434	ret
435	ENDPROC VMXDispatchHostNmi
436
437
438	%ifdef VBOX_WITH_KERNEL_USING_XMM
439
440	;;
441	; Wrapper around vmx.pfnStartVM that preserves host XMM registers and
442	; load the guest ones when necessary.
443	;
444	; @cproto DECLASM(int) hmR0VMXStartVMWrapXMM(RTHCUINT fResume, PCPUMCTX pCtx, void *pvUnused, PVM pVM,
445	; PVMCPU pVCpu, PFNHMVMXSTARTVM pfnStartVM);
446	;
447	; @returns eax
448	;
449	; @param fResumeVM msc:rcx
450	; @param pCtx msc:rdx
451	; @param pvUnused msc:r8
452	; @param pVM msc:r9
453	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
454	; @param pfnStartVM msc:[rbp+38h]
455	;
456	; @remarks This is essentially the same code as hmR0SVMRunWrapXMM, only the parameters differ a little bit.
457	;
458	; @remarks Drivers shouldn't use AVX registers without saving+loading:
459	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
460	; However the compiler docs have different idea:
461	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
462	; We'll go with the former for now.
463	;
464	; ASSUMING 64-bit and windows for now.
465	;
466	ALIGNCODE(16)
467	BEGINPROC hmR0VMXStartVMWrapXMM
468	push xBP
469	mov xBP, xSP
470	sub xSP, 0b0h + 040h ; Don't bother optimizing the frame size.
471
472	; Spill input parameters.
473	mov [xBP + 010h], rcx ; fResumeVM
474	mov [xBP + 018h], rdx ; pCtx
475	mov [xBP + 020h], r8 ; pvUnused
476	mov [xBP + 028h], r9 ; pVM
477
478	; Ask CPUM whether we've started using the FPU yet.
479	mov rcx, [xBP + 30h] ; pVCpu
480	call NAME(CPUMIsGuestFPUStateActive)
481	test al, al
482	jnz .guest_fpu_state_active
483
484	; No need to mess with XMM registers just call the start routine and return.
485	mov r11, [xBP + 38h] ; pfnStartVM
486	mov r10, [xBP + 30h] ; pVCpu
487	mov [xSP + 020h], r10
488	mov rcx, [xBP + 010h] ; fResumeVM
489	mov rdx, [xBP + 018h] ; pCtx
490	mov r8, [xBP + 020h] ; pvUnused
491	mov r9, [xBP + 028h] ; pVM
492	call r11
493
494	leave
495	ret
496
497	ALIGNCODE(8)
498	.guest_fpu_state_active:
499	; Save the non-volatile host XMM registers.
500	movdqa [rsp + 040h + 000h], xmm6
501	movdqa [rsp + 040h + 010h], xmm7
502	movdqa [rsp + 040h + 020h], xmm8
503	movdqa [rsp + 040h + 030h], xmm9
504	movdqa [rsp + 040h + 040h], xmm10
505	movdqa [rsp + 040h + 050h], xmm11
506	movdqa [rsp + 040h + 060h], xmm12
507	movdqa [rsp + 040h + 070h], xmm13
508	movdqa [rsp + 040h + 080h], xmm14
509	movdqa [rsp + 040h + 090h], xmm15
510	stmxcsr [rsp + 040h + 0a0h]
511
512	mov r10, [xBP + 018h] ; pCtx
513	mov eax, [r10 + CPUMCTX.fXStateMask]
514	test eax, eax
515	jz .guest_fpu_state_manually
516
517	;
518	; Using XSAVE to load the guest XMM, YMM and ZMM registers.
519	;
520	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
521	xor edx, edx
522	mov r10, [r10 + CPUMCTX.pXStateR0]
523	xrstor [r10]
524
525	; Make the call (same as in the other case).
526	mov r11, [xBP + 38h] ; pfnStartVM
527	mov r10, [xBP + 30h] ; pVCpu
528	mov [xSP + 020h], r10
529	mov rcx, [xBP + 010h] ; fResumeVM
530	mov rdx, [xBP + 018h] ; pCtx
531	mov r8, [xBP + 020h] ; pvUnused
532	mov r9, [xBP + 028h] ; pVM
533	call r11
534
535	mov r11d, eax ; save return value (xsave below uses eax)
536
537	; Save the guest XMM registers.
538	mov r10, [xBP + 018h] ; pCtx
539	mov eax, [r10 + CPUMCTX.fXStateMask]
540	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
541	xor edx, edx
542	mov r10, [r10 + CPUMCTX.pXStateR0]
543	xsave [r10]
544
545	mov eax, r11d ; restore return value
546
547	.restore_non_volatile_host_xmm_regs:
548	; Load the non-volatile host XMM registers.
549	movdqa xmm6, [rsp + 040h + 000h]
550	movdqa xmm7, [rsp + 040h + 010h]
551	movdqa xmm8, [rsp + 040h + 020h]
552	movdqa xmm9, [rsp + 040h + 030h]
553	movdqa xmm10, [rsp + 040h + 040h]
554	movdqa xmm11, [rsp + 040h + 050h]
555	movdqa xmm12, [rsp + 040h + 060h]
556	movdqa xmm13, [rsp + 040h + 070h]
557	movdqa xmm14, [rsp + 040h + 080h]
558	movdqa xmm15, [rsp + 040h + 090h]
559	ldmxcsr [rsp + 040h + 0a0h]
560	leave
561	ret
562
563	;
564	; No XSAVE, load and save the guest XMM registers manually.
565	;
566	.guest_fpu_state_manually:
567	; Load the full guest XMM register state.
568	mov r10, [r10 + CPUMCTX.pXStateR0]
569	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
570	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
571	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
572	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
573	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
574	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
575	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
576	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
577	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
578	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
579	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
580	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
581	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
582	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
583	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
584	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
585	ldmxcsr [r10 + X86FXSTATE.MXCSR]
586
587	; Make the call (same as in the other case).
588	mov r11, [xBP + 38h] ; pfnStartVM
589	mov r10, [xBP + 30h] ; pVCpu
590	mov [xSP + 020h], r10
591	mov rcx, [xBP + 010h] ; fResumeVM
592	mov rdx, [xBP + 018h] ; pCtx
593	mov r8, [xBP + 020h] ; pvUnused
594	mov r9, [xBP + 028h] ; pVM
595	call r11
596
597	; Save the guest XMM registers.
598	mov r10, [xBP + 018h] ; pCtx
599	mov r10, [r10 + CPUMCTX.pXStateR0]
600	stmxcsr [r10 + X86FXSTATE.MXCSR]
601	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
602	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
603	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
604	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
605	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
606	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
607	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
608	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
609	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
610	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
611	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
612	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
613	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
614	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
615	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
616	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
617	jmp .restore_non_volatile_host_xmm_regs
618	ENDPROC hmR0VMXStartVMWrapXMM
619
620	;;
621	; Wrapper around svm.pfnVMRun that preserves host XMM registers and
622	; load the guest ones when necessary.
623	;
624	; @cproto DECLASM(int) hmR0SVMRunWrapXMM(RTHCPHYS HCPhysVmcbHost, RTHCPHYS HCPhysVmcb, PCPUMCTX pCtx, PVM pVM, PVMCPU pVCpu,
625	; PFNHMSVMVMRUN pfnVMRun);
626	;
627	; @returns eax
628	;
629	; @param HCPhysVmcbHost msc:rcx
630	; @param HCPhysVmcb msc:rdx
631	; @param pCtx msc:r8
632	; @param pVM msc:r9
633	; @param pVCpu msc:[rbp+30h] The cross context virtual CPU structure of the calling EMT.
634	; @param pfnVMRun msc:[rbp+38h]
635	;
636	; @remarks This is essentially the same code as hmR0VMXStartVMWrapXMM, only the parameters differ a little bit.
637	;
638	; @remarks Drivers shouldn't use AVX registers without saving+loading:
639	; https://msdn.microsoft.com/en-us/library/windows/hardware/ff545910%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396
640	; However the compiler docs have different idea:
641	; https://msdn.microsoft.com/en-us/library/9z1stfyw.aspx
642	; We'll go with the former for now.
643	;
644	; ASSUMING 64-bit and windows for now.
645	ALIGNCODE(16)
646	BEGINPROC hmR0SVMRunWrapXMM
647	push xBP
648	mov xBP, xSP
649	sub xSP, 0b0h + 040h ; don't bother optimizing the frame size
650
651	; Spill input parameters.
652	mov [xBP + 010h], rcx ; HCPhysVmcbHost
653	mov [xBP + 018h], rdx ; HCPhysVmcb
654	mov [xBP + 020h], r8 ; pCtx
655	mov [xBP + 028h], r9 ; pVM
656
657	; Ask CPUM whether we've started using the FPU yet.
658	mov rcx, [xBP + 30h] ; pVCpu
659	call NAME(CPUMIsGuestFPUStateActive)
660	test al, al
661	jnz .guest_fpu_state_active
662
663	; No need to mess with XMM registers just call the start routine and return.
664	mov r11, [xBP + 38h] ; pfnVMRun
665	mov r10, [xBP + 30h] ; pVCpu
666	mov [xSP + 020h], r10
667	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
668	mov rdx, [xBP + 018h] ; HCPhysVmcb
669	mov r8, [xBP + 020h] ; pCtx
670	mov r9, [xBP + 028h] ; pVM
671	call r11
672
673	leave
674	ret
675
676	ALIGNCODE(8)
677	.guest_fpu_state_active:
678	; Save the non-volatile host XMM registers.
679	movdqa [rsp + 040h + 000h], xmm6
680	movdqa [rsp + 040h + 010h], xmm7
681	movdqa [rsp + 040h + 020h], xmm8
682	movdqa [rsp + 040h + 030h], xmm9
683	movdqa [rsp + 040h + 040h], xmm10
684	movdqa [rsp + 040h + 050h], xmm11
685	movdqa [rsp + 040h + 060h], xmm12
686	movdqa [rsp + 040h + 070h], xmm13
687	movdqa [rsp + 040h + 080h], xmm14
688	movdqa [rsp + 040h + 090h], xmm15
689	stmxcsr [rsp + 040h + 0a0h]
690
691	mov r10, [xBP + 020h] ; pCtx
692	mov eax, [r10 + CPUMCTX.fXStateMask]
693	test eax, eax
694	jz .guest_fpu_state_manually
695
696	;
697	; Using XSAVE.
698	;
699	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
700	xor edx, edx
701	mov r10, [r10 + CPUMCTX.pXStateR0]
702	xrstor [r10]
703
704	; Make the call (same as in the other case).
705	mov r11, [xBP + 38h] ; pfnVMRun
706	mov r10, [xBP + 30h] ; pVCpu
707	mov [xSP + 020h], r10
708	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
709	mov rdx, [xBP + 018h] ; HCPhysVmcb
710	mov r8, [xBP + 020h] ; pCtx
711	mov r9, [xBP + 028h] ; pVM
712	call r11
713
714	mov r11d, eax ; save return value (xsave below uses eax)
715
716	; Save the guest XMM registers.
717	mov r10, [xBP + 020h] ; pCtx
718	mov eax, [r10 + CPUMCTX.fXStateMask]
719	and eax, CPUM_VOLATILE_XSAVE_GUEST_COMPONENTS
720	xor edx, edx
721	mov r10, [r10 + CPUMCTX.pXStateR0]
722	xsave [r10]
723
724	mov eax, r11d ; restore return value
725
726	.restore_non_volatile_host_xmm_regs:
727	; Load the non-volatile host XMM registers.
728	movdqa xmm6, [rsp + 040h + 000h]
729	movdqa xmm7, [rsp + 040h + 010h]
730	movdqa xmm8, [rsp + 040h + 020h]
731	movdqa xmm9, [rsp + 040h + 030h]
732	movdqa xmm10, [rsp + 040h + 040h]
733	movdqa xmm11, [rsp + 040h + 050h]
734	movdqa xmm12, [rsp + 040h + 060h]
735	movdqa xmm13, [rsp + 040h + 070h]
736	movdqa xmm14, [rsp + 040h + 080h]
737	movdqa xmm15, [rsp + 040h + 090h]
738	ldmxcsr [rsp + 040h + 0a0h]
739	leave
740	ret
741
742	;
743	; No XSAVE, load and save the guest XMM registers manually.
744	;
745	.guest_fpu_state_manually:
746	; Load the full guest XMM register state.
747	mov r10, [r10 + CPUMCTX.pXStateR0]
748	movdqa xmm0, [r10 + XMM_OFF_IN_X86FXSTATE + 000h]
749	movdqa xmm1, [r10 + XMM_OFF_IN_X86FXSTATE + 010h]
750	movdqa xmm2, [r10 + XMM_OFF_IN_X86FXSTATE + 020h]
751	movdqa xmm3, [r10 + XMM_OFF_IN_X86FXSTATE + 030h]
752	movdqa xmm4, [r10 + XMM_OFF_IN_X86FXSTATE + 040h]
753	movdqa xmm5, [r10 + XMM_OFF_IN_X86FXSTATE + 050h]
754	movdqa xmm6, [r10 + XMM_OFF_IN_X86FXSTATE + 060h]
755	movdqa xmm7, [r10 + XMM_OFF_IN_X86FXSTATE + 070h]
756	movdqa xmm8, [r10 + XMM_OFF_IN_X86FXSTATE + 080h]
757	movdqa xmm9, [r10 + XMM_OFF_IN_X86FXSTATE + 090h]
758	movdqa xmm10, [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h]
759	movdqa xmm11, [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h]
760	movdqa xmm12, [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h]
761	movdqa xmm13, [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h]
762	movdqa xmm14, [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h]
763	movdqa xmm15, [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h]
764	ldmxcsr [r10 + X86FXSTATE.MXCSR]
765
766	; Make the call (same as in the other case).
767	mov r11, [xBP + 38h] ; pfnVMRun
768	mov r10, [xBP + 30h] ; pVCpu
769	mov [xSP + 020h], r10
770	mov rcx, [xBP + 010h] ; HCPhysVmcbHost
771	mov rdx, [xBP + 018h] ; HCPhysVmcb
772	mov r8, [xBP + 020h] ; pCtx
773	mov r9, [xBP + 028h] ; pVM
774	call r11
775
776	; Save the guest XMM registers.
777	mov r10, [xBP + 020h] ; pCtx
778	mov r10, [r10 + CPUMCTX.pXStateR0]
779	stmxcsr [r10 + X86FXSTATE.MXCSR]
780	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 000h], xmm0
781	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 010h], xmm1
782	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 020h], xmm2
783	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 030h], xmm3
784	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 040h], xmm4
785	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 050h], xmm5
786	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 060h], xmm6
787	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 070h], xmm7
788	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 080h], xmm8
789	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 090h], xmm9
790	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0a0h], xmm10
791	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0b0h], xmm11
792	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0c0h], xmm12
793	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0d0h], xmm13
794	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0e0h], xmm14
795	movdqa [r10 + XMM_OFF_IN_X86FXSTATE + 0f0h], xmm15
796	jmp .restore_non_volatile_host_xmm_regs
797	ENDPROC hmR0SVMRunWrapXMM
798
799	%endif ; VBOX_WITH_KERNEL_USING_XMM
800
801
802	%ifdef RT_ARCH_AMD64
803	;; @def RESTORE_STATE_VM64
804	; Macro restoring essential host state and updating guest state
805	; for 64-bit host, 64-bit guest for VT-x.
806	;
807	%macro RESTORE_STATE_VM64 0
808	; Restore base and limit of the IDTR & GDTR.
809	%ifndef VMX_SKIP_IDTR
810	lidt [xSP]
811	add xSP, xCB * 2
812	%endif
813	%ifndef VMX_SKIP_GDTR
814	lgdt [xSP]
815	add xSP, xCB * 2
816	%endif
817
818	push xDI
819	%ifndef VMX_SKIP_TR
820	mov xDI, [xSP + xCB * 3] ; pCtx (*3 to skip the saved xDI, TR, LDTR)
821	%else
822	mov xDI, [xSP + xCB * 2] ; pCtx (*2 to skip the saved xDI, LDTR)
823	%endif
824
825	mov qword [xDI + CPUMCTX.eax], rax
826	mov rax, SPECTRE_FILLER64
827	mov qword [xDI + CPUMCTX.ebx], rbx
828	mov rbx, rax
829	mov qword [xDI + CPUMCTX.ecx], rcx
830	mov rcx, rax
831	mov qword [xDI + CPUMCTX.edx], rdx
832	mov rdx, rax
833	mov qword [xDI + CPUMCTX.esi], rsi
834	mov rsi, rax
835	mov qword [xDI + CPUMCTX.ebp], rbp
836	mov rbp, rax
837	mov qword [xDI + CPUMCTX.r8], r8
838	mov r8, rax
839	mov qword [xDI + CPUMCTX.r9], r9
840	mov r9, rax
841	mov qword [xDI + CPUMCTX.r10], r10
842	mov r10, rax
843	mov qword [xDI + CPUMCTX.r11], r11
844	mov r11, rax
845	mov qword [xDI + CPUMCTX.r12], r12
846	mov r12, rax
847	mov qword [xDI + CPUMCTX.r13], r13
848	mov r13, rax
849	mov qword [xDI + CPUMCTX.r14], r14
850	mov r14, rax
851	mov qword [xDI + CPUMCTX.r15], r15
852	mov r15, rax
853	mov rax, cr2
854	mov qword [xDI + CPUMCTX.cr2], rax
855
856	pop xAX ; The guest rdi we pushed above
857	mov qword [xDI + CPUMCTX.edi], rax
858
859	; Fight spectre.
860	INDIRECT_BRANCH_PREDICTION_BARRIER xDI, CPUMCTX_WSF_IBPB_EXIT
861
862	%ifndef VMX_SKIP_TR
863	; Restore TSS selector; must mark it as not busy before using ltr!
864	; ASSUME that this is supposed to be 'BUSY' (saves 20-30 ticks on the T42p).
865	; @todo get rid of sgdt
866	pop xBX ; Saved TR
867	sub xSP, xCB * 2
868	sgdt [xSP]
869	mov xAX, xBX
870	and eax, X86_SEL_MASK_OFF_RPL ; mask away TI and RPL bits leaving only the descriptor offset
871	add xAX, [xSP + 2] ; eax <- GDTR.address + descriptor offset
872	and dword [xAX + 4], ~RT_BIT(9) ; clear the busy flag in TSS desc (bits 0-7=base, bit 9=busy bit)
873	ltr bx
874	add xSP, xCB * 2
875	%endif
876
877	pop xAX ; Saved LDTR
878	cmp eax, 0
879	je %%skip_ldt_write64
880	lldt ax
881
882	%%skip_ldt_write64:
883	pop xSI ; pCtx (needed in rsi by the macros below)
884
885	; Restore segment registers.
886	MYPOPSEGS xAX, ax
887
888	; Restore the host XCR0 if necessary.
889	pop xCX
890	test ecx, ecx
891	jnz %%xcr0_after_skip
892	pop xAX
893	pop xDX
894	xsetbv ; ecx is already zero.
895	%%xcr0_after_skip:
896
897	; Restore general purpose registers.
898	MYPOPAD
899	%endmacro
900
901
902	;;
903	; Prepares for and executes VMLAUNCH/VMRESUME (64 bits guest mode)
904	;
905	; @returns VBox status code
906	; @param fResume msc:rcx, gcc:rdi Whether to use vmlauch/vmresume.
907	; @param pCtx msc:rdx, gcc:rsi Pointer to the guest-CPU context.
908	; @param pvUnused msc:r8, gcc:rdx Unused argument.
909	; @param pVM msc:r9, gcc:rcx The cross context VM structure.
910	; @param pVCpu msc:[ebp+30], gcc:r8 The cross context virtual CPU structure of the calling EMT.
911	;
912	ALIGNCODE(16)
913	BEGINPROC VMXR0StartVM64
914	push xBP
915	mov xBP, xSP
916
917	pushf
918	cli
919
920	; Save all general purpose host registers.
921	MYPUSHAD
922
923	; First we have to save some final CPU context registers.
924	lea r10, [.vmlaunch64_done wrt rip]
925	mov rax, VMX_VMCS_HOST_RIP ; return address (too difficult to continue after VMLAUNCH?)
926	vmwrite rax, r10
927	; Note: ASSUMES success!
928
929	;
930	; Unify the input parameter registers.
931	;
932	%ifdef ASM_CALL64_GCC
933	; fResume already in rdi
934	; pCtx already in rsi
935	mov rbx, rdx ; pvUnused
936	%else
937	mov rdi, rcx ; fResume
938	mov rsi, rdx ; pCtx
939	mov rbx, r8 ; pvUnused
940	%endif
941
942	;
943	; Save the host XCR0 and load the guest one if necessary.
944	; Note! Trashes rdx and rcx.
945	;
946	%ifdef ASM_CALL64_MSC
947	mov rax, [xBP + 30h] ; pVCpu
948	%else
949	mov rax, r8 ; pVCpu
950	%endif
951	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
952	jz .xcr0_before_skip
953
954	xor ecx, ecx
955	xgetbv ; save the host one on the stack
956	push xDX
957	push xAX
958
959	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest one
960	mov edx, [xSI + CPUMCTX.aXcr + 4]
961	xor ecx, ecx ; paranoia
962	xsetbv
963
964	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
965	jmp .xcr0_before_done
966
967	.xcr0_before_skip:
968	push 3fh ; indicate that we need not
969	.xcr0_before_done:
970
971	;
972	; Save segment registers.
973	; Note! Trashes rdx & rcx, so we moved it here (amd64 case).
974	;
975	MYPUSHSEGS xAX, ax
976
977	; Save the pCtx pointer.
978	push xSI
979
980	; Save host LDTR.
981	xor eax, eax
982	sldt ax
983	push xAX
984
985	%ifndef VMX_SKIP_TR
986	; The host TR limit is reset to 0x67; save & restore it manually.
987	str eax
988	push xAX
989	%endif
990
991	%ifndef VMX_SKIP_GDTR
992	; VT-x only saves the base of the GDTR & IDTR and resets the limit to 0xffff; we must restore the limit correctly!
993	sub xSP, xCB * 2
994	sgdt [xSP]
995	%endif
996	%ifndef VMX_SKIP_IDTR
997	sub xSP, xCB * 2
998	sidt [xSP]
999	%endif
1000
1001	; Load CR2 if necessary (may be expensive as writing CR2 is a synchronizing instruction).
1002	mov rbx, qword [xSI + CPUMCTX.cr2]
1003	mov rdx, cr2
1004	cmp rbx, rdx
1005	je .skip_cr2_write
1006	mov cr2, rbx
1007
1008	.skip_cr2_write:
1009	mov eax, VMX_VMCS_HOST_RSP
1010	vmwrite xAX, xSP
1011	; Note: ASSUMES success!
1012	; Don't mess with ESP anymore!!!
1013
1014	; Fight spectre and similar.
1015	INDIRECT_BRANCH_PREDICTION_AND_L1_CACHE_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY, CPUMCTX_WSF_L1D_ENTRY, CPUMCTX_WSF_MDS_ENTRY
1016
1017	; Load guest general purpose registers.
1018	mov rax, qword [xSI + CPUMCTX.eax]
1019	mov rbx, qword [xSI + CPUMCTX.ebx]
1020	mov rcx, qword [xSI + CPUMCTX.ecx]
1021	mov rdx, qword [xSI + CPUMCTX.edx]
1022	mov rbp, qword [xSI + CPUMCTX.ebp]
1023	mov r8, qword [xSI + CPUMCTX.r8]
1024	mov r9, qword [xSI + CPUMCTX.r9]
1025	mov r10, qword [xSI + CPUMCTX.r10]
1026	mov r11, qword [xSI + CPUMCTX.r11]
1027	mov r12, qword [xSI + CPUMCTX.r12]
1028	mov r13, qword [xSI + CPUMCTX.r13]
1029	mov r14, qword [xSI + CPUMCTX.r14]
1030	mov r15, qword [xSI + CPUMCTX.r15]
1031
1032	; Resume or start VM?
1033	cmp xDI, 0 ; fResume
1034
1035	; Load guest rdi & rsi.
1036	mov rdi, qword [xSI + CPUMCTX.edi]
1037	mov rsi, qword [xSI + CPUMCTX.esi]
1038
1039	je .vmlaunch64_launch
1040
1041	vmresume
1042	jc near .vmxstart64_invalid_vmcs_ptr
1043	jz near .vmxstart64_start_failed
1044	jmp .vmlaunch64_done; ; here if vmresume detected a failure
1045
1046	.vmlaunch64_launch:
1047	vmlaunch
1048	jc near .vmxstart64_invalid_vmcs_ptr
1049	jz near .vmxstart64_start_failed
1050	jmp .vmlaunch64_done; ; here if vmlaunch detected a failure
1051
1052	ALIGNCODE(16)
1053	.vmlaunch64_done:
1054	RESTORE_STATE_VM64
1055	mov eax, VINF_SUCCESS
1056
1057	.vmstart64_end:
1058	popf
1059	pop xBP
1060	ret
1061
1062	.vmxstart64_invalid_vmcs_ptr:
1063	RESTORE_STATE_VM64
1064	mov eax, VERR_VMX_INVALID_VMCS_PTR_TO_START_VM
1065	jmp .vmstart64_end
1066
1067	.vmxstart64_start_failed:
1068	RESTORE_STATE_VM64
1069	mov eax, VERR_VMX_UNABLE_TO_START_VM
1070	jmp .vmstart64_end
1071	ENDPROC VMXR0StartVM64
1072	%endif ; RT_ARCH_AMD64
1073
1074
1075	;;
1076	; Clears the MDS buffers using VERW.
1077	ALIGNCODE(16)
1078	BEGINPROC hmR0MdsClear
1079	sub xSP, xCB
1080	mov [xSP], ds
1081	verw [xSP]
1082	add xSP, xCB
1083	ret
1084	ENDPROC hmR0MdsClear
1085
1086
1087	%ifdef RT_ARCH_AMD64
1088	;;
1089	; Prepares for and executes VMRUN (32-bit and 64-bit guests).
1090	;
1091	; @returns VBox status code
1092	; @param HCPhysVmcbHost msc:rcx,gcc:rdi Physical address of host VMCB.
1093	; @param HCPhysVmcb msc:rdx,gcc:rsi Physical address of guest VMCB.
1094	; @param pCtx msc:r8,gcc:rdx Pointer to the guest-CPU context.
1095	; @param pVM msc:r9,gcc:rcx The cross context VM structure.
1096	; @param pVCpu msc:[rsp+28],gcc:r8 The cross context virtual CPU structure of the calling EMT.
1097	;
1098	ALIGNCODE(16)
1099	BEGINPROC SVMR0VMRun
1100	; Fake a cdecl stack frame
1101	%ifdef ASM_CALL64_GCC
1102	push r8 ; pVCpu
1103	push rcx ; pVM
1104	push rdx ; pCtx
1105	push rsi ; HCPhysVmcb
1106	push rdi ; HCPhysVmcbHost
1107	%else
1108	mov rax, [rsp + 28h]
1109	push rax ; rbp + 30h pVCpu
1110	push r9 ; rbp + 28h pVM
1111	push r8 ; rbp + 20h pCtx
1112	push rdx ; rbp + 18h HCPhysVmcb
1113	push rcx ; rbp + 10h HCPhysVmcbHost
1114	%endif
1115	push 0 ; rbp + 08h "fake ret addr"
1116	push rbp ; rbp + 00h
1117	mov rbp, rsp
1118	pushf
1119
1120	; Manual save and restore:
1121	; - General purpose registers except RIP, RSP, RAX
1122	;
1123	; Trashed:
1124	; - CR2 (we don't care)
1125	; - LDTR (reset to 0)
1126	; - DRx (presumably not changed at all)
1127	; - DR7 (reset to 0x400)
1128
1129	; Save all general purpose host registers.
1130	MYPUSHAD
1131
1132	; Load pCtx into xSI.
1133	mov xSI, [rbp + xCB * 2 + RTHCPHYS_CB * 2]
1134
1135	; Save the host XCR0 and load the guest one if necessary.
1136	mov rax, [xBP + 30h] ; pVCpu
1137	test byte [xAX + VMCPU.hm + HMCPU.fLoadSaveGuestXcr0], 1
1138	jz .xcr0_before_skip
1139
1140	xor ecx, ecx
1141	xgetbv ; save the host XCR0 on the stack
1142	push xDX
1143	push xAX
1144
1145	mov xSI, [xBP + xCB * 2 + RTHCPHYS_CB * 2] ; pCtx
1146	mov eax, [xSI + CPUMCTX.aXcr] ; load the guest XCR0
1147	mov edx, [xSI + CPUMCTX.aXcr + 4]
1148	xor ecx, ecx ; paranoia
1149	xsetbv
1150
1151	push 0 ; indicate that we must restore XCR0 (popped into ecx, thus 0)
1152	jmp .xcr0_before_done
1153
1154	.xcr0_before_skip:
1155	push 3fh ; indicate that we need not restore XCR0
1156	.xcr0_before_done:
1157
1158	; Save guest CPU-context pointer for simplifying saving of the GPRs afterwards.
1159	push rsi
1160
1161	; Save host fs, gs, sysenter msr etc.
1162	mov rax, [rbp + xCB * 2] ; HCPhysVmcbHost (64 bits physical address; x86: take low dword only)
1163	push rax ; save for the vmload after vmrun
1164	vmsave
1165
1166	; Fight spectre.
1167	INDIRECT_BRANCH_PREDICTION_BARRIER xSI, CPUMCTX_WSF_IBPB_ENTRY
1168
1169	; Setup rax for VMLOAD.
1170	mov rax, [rbp + xCB * 2 + RTHCPHYS_CB] ; HCPhysVmcb (64 bits physical address; take low dword only)
1171
1172	; Load guest general purpose registers (rax is loaded from the VMCB by VMRUN).
1173	mov rbx, qword [xSI + CPUMCTX.ebx]
1174	mov rcx, qword [xSI + CPUMCTX.ecx]
1175	mov rdx, qword [xSI + CPUMCTX.edx]
1176	mov rdi, qword [xSI + CPUMCTX.edi]
1177	mov rbp, qword [xSI + CPUMCTX.ebp]
1178	mov r8, qword [xSI + CPUMCTX.r8]
1179	mov r9, qword [xSI + CPUMCTX.r9]
1180	mov r10, qword [xSI + CPUMCTX.r10]
1181	mov r11, qword [xSI + CPUMCTX.r11]
1182	mov r12, qword [xSI + CPUMCTX.r12]
1183	mov r13, qword [xSI + CPUMCTX.r13]
1184	mov r14, qword [xSI + CPUMCTX.r14]
1185	mov r15, qword [xSI + CPUMCTX.r15]
1186	mov rsi, qword [xSI + CPUMCTX.esi]
1187
1188	; Clear the global interrupt flag & execute sti to make sure external interrupts cause a world switch.
1189	clgi
1190	sti
1191
1192	; Load guest FS, GS, Sysenter MSRs etc.
1193	vmload
1194
1195	; Run the VM.
1196	vmrun
1197
1198	; Save guest fs, gs, sysenter msr etc.
1199	vmsave
1200
1201	; Load host fs, gs, sysenter msr etc.
1202	pop rax ; load HCPhysVmcbHost (pushed above)
1203	vmload
1204
1205	; Set the global interrupt flag again, but execute cli to make sure IF=0.
1206	cli
1207	stgi
1208
1209	; Pop the context pointer (pushed above) and save the guest GPRs (sans RSP and RAX).
1210	pop rax
1211
1212	mov qword [rax + CPUMCTX.ebx], rbx
1213	mov rbx, SPECTRE_FILLER64
1214	mov qword [rax + CPUMCTX.ecx], rcx
1215	mov rcx, rbx
1216	mov qword [rax + CPUMCTX.edx], rdx
1217	mov rdx, rbx
1218	mov qword [rax + CPUMCTX.esi], rsi
1219	mov rsi, rbx
1220	mov qword [rax + CPUMCTX.edi], rdi
1221	mov rdi, rbx
1222	mov qword [rax + CPUMCTX.ebp], rbp
1223	mov rbp, rbx
1224	mov qword [rax + CPUMCTX.r8], r8
1225	mov r8, rbx
1226	mov qword [rax + CPUMCTX.r9], r9
1227	mov r9, rbx
1228	mov qword [rax + CPUMCTX.r10], r10
1229	mov r10, rbx
1230	mov qword [rax + CPUMCTX.r11], r11
1231	mov r11, rbx
1232	mov qword [rax + CPUMCTX.r12], r12
1233	mov r12, rbx
1234	mov qword [rax + CPUMCTX.r13], r13
1235	mov r13, rbx
1236	mov qword [rax + CPUMCTX.r14], r14
1237	mov r14, rbx
1238	mov qword [rax + CPUMCTX.r15], r15
1239	mov r15, rbx
1240
1241	; Fight spectre. Note! Trashes rax!
1242	INDIRECT_BRANCH_PREDICTION_BARRIER rax, CPUMCTX_WSF_IBPB_EXIT
1243
1244	; Restore the host xcr0 if necessary.
1245	pop xCX
1246	test ecx, ecx
1247	jnz .xcr0_after_skip
1248	pop xAX
1249	pop xDX
1250	xsetbv ; ecx is already zero
1251	.xcr0_after_skip:
1252
1253	; Restore host general purpose registers.
1254	MYPOPAD
1255
1256	mov eax, VINF_SUCCESS
1257
1258	popf
1259	pop rbp
1260	add rsp, 6 * xCB
1261	ret
1262	ENDPROC SVMR0VMRun
1263	%endif ; RT_ARCH_AMD64
1264

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMR0A.asm@ 87310

Download in other formats: