PGMAll.cpp@ 91712

Last change on this file since 91712 was 91712, checked in by vboxsync, 4 years ago
VMM/PGM: Nested VMX: bugref:10092 Removed PGMMODEDATAGST::pfnGetPDE, no longer used.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 136.1 KB

Line
1	/* $Id: PGMAll.cpp 91712 2021-10-13 11:33:18Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM
23	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
24	#include <VBox/vmm/pgm.h>
25	#include <VBox/vmm/cpum.h>
26	#include <VBox/vmm/selm.h>
27	#include <VBox/vmm/iem.h>
28	#include <VBox/vmm/iom.h>
29	#include <VBox/sup.h>
30	#include <VBox/vmm/mm.h>
31	#include <VBox/vmm/stam.h>
32	#include <VBox/vmm/trpm.h>
33	#include <VBox/vmm/em.h>
34	#include <VBox/vmm/hm.h>
35	#include <VBox/vmm/hm_vmx.h>
36	#include "PGMInternal.h"
37	#include <VBox/vmm/vmcc.h>
38	#include "PGMInline.h"
39	#include <iprt/assert.h>
40	#include <iprt/asm-amd64-x86.h>
41	#include <iprt/string.h>
42	#include <VBox/log.h>
43	#include <VBox/param.h>
44	#include <VBox/err.h>
45
46
47	/*********************************************************************************************************************************
48	* Internal Functions *
49	*********************************************************************************************************************************/
50	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
51	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
52	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
53	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
54
55
56	/*
57	* Shadow - 32-bit mode
58	*/
59	#define PGM_SHW_TYPE PGM_TYPE_32BIT
60	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
61	#include "PGMAllShw.h"
62
63	/* Guest - real mode */
64	#define PGM_GST_TYPE PGM_TYPE_REAL
65	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
66	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
67	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
68	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
69	#include "PGMGstDefs.h"
70	#include "PGMAllGst.h"
71	#include "PGMAllBth.h"
72	#undef BTH_PGMPOOLKIND_PT_FOR_PT
73	#undef BTH_PGMPOOLKIND_ROOT
74	#undef PGM_BTH_NAME
75	#undef PGM_GST_TYPE
76	#undef PGM_GST_NAME
77
78	/* Guest - protected mode */
79	#define PGM_GST_TYPE PGM_TYPE_PROT
80	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
81	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
82	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
83	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
84	#include "PGMGstDefs.h"
85	#include "PGMAllGst.h"
86	#include "PGMAllBth.h"
87	#undef BTH_PGMPOOLKIND_PT_FOR_PT
88	#undef BTH_PGMPOOLKIND_ROOT
89	#undef PGM_BTH_NAME
90	#undef PGM_GST_TYPE
91	#undef PGM_GST_NAME
92
93	/* Guest - 32-bit mode */
94	#define PGM_GST_TYPE PGM_TYPE_32BIT
95	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
96	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
97	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
98	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
99	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
100	#include "PGMGstDefs.h"
101	#include "PGMAllGst.h"
102	#include "PGMAllBth.h"
103	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
104	#undef BTH_PGMPOOLKIND_PT_FOR_PT
105	#undef BTH_PGMPOOLKIND_ROOT
106	#undef PGM_BTH_NAME
107	#undef PGM_GST_TYPE
108	#undef PGM_GST_NAME
109
110	#undef PGM_SHW_TYPE
111	#undef PGM_SHW_NAME
112
113
114	/*
115	* Shadow - PAE mode
116	*/
117	#define PGM_SHW_TYPE PGM_TYPE_PAE
118	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
119	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
120	#include "PGMAllShw.h"
121
122	/* Guest - real mode */
123	#define PGM_GST_TYPE PGM_TYPE_REAL
124	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
125	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
126	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
127	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
128	#include "PGMGstDefs.h"
129	#include "PGMAllBth.h"
130	#undef BTH_PGMPOOLKIND_PT_FOR_PT
131	#undef BTH_PGMPOOLKIND_ROOT
132	#undef PGM_BTH_NAME
133	#undef PGM_GST_TYPE
134	#undef PGM_GST_NAME
135
136	/* Guest - protected mode */
137	#define PGM_GST_TYPE PGM_TYPE_PROT
138	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
139	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
140	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
141	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
142	#include "PGMGstDefs.h"
143	#include "PGMAllBth.h"
144	#undef BTH_PGMPOOLKIND_PT_FOR_PT
145	#undef BTH_PGMPOOLKIND_ROOT
146	#undef PGM_BTH_NAME
147	#undef PGM_GST_TYPE
148	#undef PGM_GST_NAME
149
150	/* Guest - 32-bit mode */
151	#define PGM_GST_TYPE PGM_TYPE_32BIT
152	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
153	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
154	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
155	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
156	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
157	#include "PGMGstDefs.h"
158	#include "PGMAllBth.h"
159	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
160	#undef BTH_PGMPOOLKIND_PT_FOR_PT
161	#undef BTH_PGMPOOLKIND_ROOT
162	#undef PGM_BTH_NAME
163	#undef PGM_GST_TYPE
164	#undef PGM_GST_NAME
165
166
167	/* Guest - PAE mode */
168	#define PGM_GST_TYPE PGM_TYPE_PAE
169	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
170	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
171	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
172	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
173	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
174	#include "PGMGstDefs.h"
175	#include "PGMAllGst.h"
176	#include "PGMAllBth.h"
177	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
178	#undef BTH_PGMPOOLKIND_PT_FOR_PT
179	#undef BTH_PGMPOOLKIND_ROOT
180	#undef PGM_BTH_NAME
181	#undef PGM_GST_TYPE
182	#undef PGM_GST_NAME
183
184	#undef PGM_SHW_TYPE
185	#undef PGM_SHW_NAME
186
187
188	/*
189	* Shadow - AMD64 mode
190	*/
191	#define PGM_SHW_TYPE PGM_TYPE_AMD64
192	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
193	#include "PGMAllShw.h"
194
195	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
196	/** @todo retire this hack. */
197	#define PGM_GST_TYPE PGM_TYPE_PROT
198	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
199	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
200	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
201	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
202	#include "PGMGstDefs.h"
203	#include "PGMAllBth.h"
204	#undef BTH_PGMPOOLKIND_PT_FOR_PT
205	#undef BTH_PGMPOOLKIND_ROOT
206	#undef PGM_BTH_NAME
207	#undef PGM_GST_TYPE
208	#undef PGM_GST_NAME
209
210	#ifdef VBOX_WITH_64_BITS_GUESTS
211	/* Guest - AMD64 mode */
212	# define PGM_GST_TYPE PGM_TYPE_AMD64
213	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
214	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
215	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
216	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
217	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
218	# include "PGMGstDefs.h"
219	# include "PGMAllGst.h"
220	# include "PGMAllBth.h"
221	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
222	# undef BTH_PGMPOOLKIND_PT_FOR_PT
223	# undef BTH_PGMPOOLKIND_ROOT
224	# undef PGM_BTH_NAME
225	# undef PGM_GST_TYPE
226	# undef PGM_GST_NAME
227	#endif /* VBOX_WITH_64_BITS_GUESTS */
228
229	#undef PGM_SHW_TYPE
230	#undef PGM_SHW_NAME
231
232
233	/*
234	* Shadow - 32-bit nested paging mode.
235	*/
236	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
237	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
238	#include "PGMAllShw.h"
239
240	/* Guest - real mode */
241	#define PGM_GST_TYPE PGM_TYPE_REAL
242	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
243	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
244	#include "PGMGstDefs.h"
245	#include "PGMAllBth.h"
246	#undef PGM_BTH_NAME
247	#undef PGM_GST_TYPE
248	#undef PGM_GST_NAME
249
250	/* Guest - protected mode */
251	#define PGM_GST_TYPE PGM_TYPE_PROT
252	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
253	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
254	#include "PGMGstDefs.h"
255	#include "PGMAllBth.h"
256	#undef PGM_BTH_NAME
257	#undef PGM_GST_TYPE
258	#undef PGM_GST_NAME
259
260	/* Guest - 32-bit mode */
261	#define PGM_GST_TYPE PGM_TYPE_32BIT
262	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
263	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
264	#include "PGMGstDefs.h"
265	#include "PGMAllBth.h"
266	#undef PGM_BTH_NAME
267	#undef PGM_GST_TYPE
268	#undef PGM_GST_NAME
269
270	/* Guest - PAE mode */
271	#define PGM_GST_TYPE PGM_TYPE_PAE
272	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
273	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
274	#include "PGMGstDefs.h"
275	#include "PGMAllBth.h"
276	#undef PGM_BTH_NAME
277	#undef PGM_GST_TYPE
278	#undef PGM_GST_NAME
279
280	#ifdef VBOX_WITH_64_BITS_GUESTS
281	/* Guest - AMD64 mode */
282	# define PGM_GST_TYPE PGM_TYPE_AMD64
283	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
284	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
285	# include "PGMGstDefs.h"
286	# include "PGMAllBth.h"
287	# undef PGM_BTH_NAME
288	# undef PGM_GST_TYPE
289	# undef PGM_GST_NAME
290	#endif /* VBOX_WITH_64_BITS_GUESTS */
291
292	#undef PGM_SHW_TYPE
293	#undef PGM_SHW_NAME
294
295
296	/*
297	* Shadow - PAE nested paging mode.
298	*/
299	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
300	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
301	#include "PGMAllShw.h"
302
303	/* Guest - real mode */
304	#define PGM_GST_TYPE PGM_TYPE_REAL
305	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
306	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
307	#include "PGMGstDefs.h"
308	#include "PGMAllBth.h"
309	#undef PGM_BTH_NAME
310	#undef PGM_GST_TYPE
311	#undef PGM_GST_NAME
312
313	/* Guest - protected mode */
314	#define PGM_GST_TYPE PGM_TYPE_PROT
315	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
316	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
317	#include "PGMGstDefs.h"
318	#include "PGMAllBth.h"
319	#undef PGM_BTH_NAME
320	#undef PGM_GST_TYPE
321	#undef PGM_GST_NAME
322
323	/* Guest - 32-bit mode */
324	#define PGM_GST_TYPE PGM_TYPE_32BIT
325	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
326	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
327	#include "PGMGstDefs.h"
328	#include "PGMAllBth.h"
329	#undef PGM_BTH_NAME
330	#undef PGM_GST_TYPE
331	#undef PGM_GST_NAME
332
333	/* Guest - PAE mode */
334	#define PGM_GST_TYPE PGM_TYPE_PAE
335	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
336	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
337	#include "PGMGstDefs.h"
338	#include "PGMAllBth.h"
339	#undef PGM_BTH_NAME
340	#undef PGM_GST_TYPE
341	#undef PGM_GST_NAME
342
343	#ifdef VBOX_WITH_64_BITS_GUESTS
344	/* Guest - AMD64 mode */
345	# define PGM_GST_TYPE PGM_TYPE_AMD64
346	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
347	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
348	# include "PGMGstDefs.h"
349	# include "PGMAllBth.h"
350	# undef PGM_BTH_NAME
351	# undef PGM_GST_TYPE
352	# undef PGM_GST_NAME
353	#endif /* VBOX_WITH_64_BITS_GUESTS */
354
355	#undef PGM_SHW_TYPE
356	#undef PGM_SHW_NAME
357
358
359	/*
360	* Shadow - AMD64 nested paging mode.
361	*/
362	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
363	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
364	#include "PGMAllShw.h"
365
366	/* Guest - real mode */
367	#define PGM_GST_TYPE PGM_TYPE_REAL
368	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
369	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
370	#include "PGMGstDefs.h"
371	#include "PGMAllBth.h"
372	#undef PGM_BTH_NAME
373	#undef PGM_GST_TYPE
374	#undef PGM_GST_NAME
375
376	/* Guest - protected mode */
377	#define PGM_GST_TYPE PGM_TYPE_PROT
378	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
379	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
380	#include "PGMGstDefs.h"
381	#include "PGMAllBth.h"
382	#undef PGM_BTH_NAME
383	#undef PGM_GST_TYPE
384	#undef PGM_GST_NAME
385
386	/* Guest - 32-bit mode */
387	#define PGM_GST_TYPE PGM_TYPE_32BIT
388	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
389	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
390	#include "PGMGstDefs.h"
391	#include "PGMAllBth.h"
392	#undef PGM_BTH_NAME
393	#undef PGM_GST_TYPE
394	#undef PGM_GST_NAME
395
396	/* Guest - PAE mode */
397	#define PGM_GST_TYPE PGM_TYPE_PAE
398	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
399	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
400	#include "PGMGstDefs.h"
401	#include "PGMAllBth.h"
402	#undef PGM_BTH_NAME
403	#undef PGM_GST_TYPE
404	#undef PGM_GST_NAME
405
406	#ifdef VBOX_WITH_64_BITS_GUESTS
407	/* Guest - AMD64 mode */
408	# define PGM_GST_TYPE PGM_TYPE_AMD64
409	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
410	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
411	# include "PGMGstDefs.h"
412	# include "PGMAllBth.h"
413	# undef PGM_BTH_NAME
414	# undef PGM_GST_TYPE
415	# undef PGM_GST_NAME
416	#endif /* VBOX_WITH_64_BITS_GUESTS */
417
418	#undef PGM_SHW_TYPE
419	#undef PGM_SHW_NAME
420
421
422	/*
423	* Shadow - EPT.
424	*/
425	#define PGM_SHW_TYPE PGM_TYPE_EPT
426	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
427	#include "PGMAllShw.h"
428
429	/* Guest - real mode */
430	#define PGM_GST_TYPE PGM_TYPE_REAL
431	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
432	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
433	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
434	#include "PGMGstDefs.h"
435	#include "PGMAllBth.h"
436	#undef BTH_PGMPOOLKIND_PT_FOR_PT
437	#undef PGM_BTH_NAME
438	#undef PGM_GST_TYPE
439	#undef PGM_GST_NAME
440
441	/* Guest - protected mode */
442	#define PGM_GST_TYPE PGM_TYPE_PROT
443	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
444	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
445	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
446	#include "PGMGstDefs.h"
447	#include "PGMAllBth.h"
448	#undef BTH_PGMPOOLKIND_PT_FOR_PT
449	#undef PGM_BTH_NAME
450	#undef PGM_GST_TYPE
451	#undef PGM_GST_NAME
452
453	/* Guest - 32-bit mode */
454	#define PGM_GST_TYPE PGM_TYPE_32BIT
455	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
456	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
457	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
458	#include "PGMGstDefs.h"
459	#include "PGMAllBth.h"
460	#undef BTH_PGMPOOLKIND_PT_FOR_PT
461	#undef PGM_BTH_NAME
462	#undef PGM_GST_TYPE
463	#undef PGM_GST_NAME
464
465	/* Guest - PAE mode */
466	#define PGM_GST_TYPE PGM_TYPE_PAE
467	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
468	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
469	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
470	#include "PGMGstDefs.h"
471	#include "PGMAllBth.h"
472	#undef BTH_PGMPOOLKIND_PT_FOR_PT
473	#undef PGM_BTH_NAME
474	#undef PGM_GST_TYPE
475	#undef PGM_GST_NAME
476
477	#ifdef VBOX_WITH_64_BITS_GUESTS
478	/* Guest - AMD64 mode */
479	# define PGM_GST_TYPE PGM_TYPE_AMD64
480	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
481	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
482	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
483	# include "PGMGstDefs.h"
484	# include "PGMAllBth.h"
485	# undef BTH_PGMPOOLKIND_PT_FOR_PT
486	# undef PGM_BTH_NAME
487	# undef PGM_GST_TYPE
488	# undef PGM_GST_NAME
489	#endif /* VBOX_WITH_64_BITS_GUESTS */
490
491	#undef PGM_SHW_TYPE
492	#undef PGM_SHW_NAME
493
494
495	/*
496	* Shadow - NEM / None.
497	*/
498	#define PGM_SHW_TYPE PGM_TYPE_NONE
499	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
500	#include "PGMAllShw.h"
501
502	/* Guest - real mode */
503	#define PGM_GST_TYPE PGM_TYPE_REAL
504	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
505	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
506	#include "PGMGstDefs.h"
507	#include "PGMAllBth.h"
508	#undef PGM_BTH_NAME
509	#undef PGM_GST_TYPE
510	#undef PGM_GST_NAME
511
512	/* Guest - protected mode */
513	#define PGM_GST_TYPE PGM_TYPE_PROT
514	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
515	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
516	#include "PGMGstDefs.h"
517	#include "PGMAllBth.h"
518	#undef PGM_BTH_NAME
519	#undef PGM_GST_TYPE
520	#undef PGM_GST_NAME
521
522	/* Guest - 32-bit mode */
523	#define PGM_GST_TYPE PGM_TYPE_32BIT
524	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
525	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
526	#include "PGMGstDefs.h"
527	#include "PGMAllBth.h"
528	#undef PGM_BTH_NAME
529	#undef PGM_GST_TYPE
530	#undef PGM_GST_NAME
531
532	/* Guest - PAE mode */
533	#define PGM_GST_TYPE PGM_TYPE_PAE
534	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
535	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
536	#include "PGMGstDefs.h"
537	#include "PGMAllBth.h"
538	#undef PGM_BTH_NAME
539	#undef PGM_GST_TYPE
540	#undef PGM_GST_NAME
541
542	#ifdef VBOX_WITH_64_BITS_GUESTS
543	/* Guest - AMD64 mode */
544	# define PGM_GST_TYPE PGM_TYPE_AMD64
545	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
546	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
547	# include "PGMGstDefs.h"
548	# include "PGMAllBth.h"
549	# undef PGM_BTH_NAME
550	# undef PGM_GST_TYPE
551	# undef PGM_GST_NAME
552	#endif /* VBOX_WITH_64_BITS_GUESTS */
553
554	#undef PGM_SHW_TYPE
555	#undef PGM_SHW_NAME
556
557
558
559	/**
560	* Guest mode data array.
561	*/
562	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
563	{
564	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
565	{
566	PGM_TYPE_REAL,
567	PGM_GST_NAME_REAL(GetPage),
568	PGM_GST_NAME_REAL(ModifyPage),
569	PGM_GST_NAME_REAL(Enter),
570	PGM_GST_NAME_REAL(Exit),
571	#ifdef IN_RING3
572	PGM_GST_NAME_REAL(Relocate),
573	#endif
574	},
575	{
576	PGM_TYPE_PROT,
577	PGM_GST_NAME_PROT(GetPage),
578	PGM_GST_NAME_PROT(ModifyPage),
579	PGM_GST_NAME_PROT(Enter),
580	PGM_GST_NAME_PROT(Exit),
581	#ifdef IN_RING3
582	PGM_GST_NAME_PROT(Relocate),
583	#endif
584	},
585	{
586	PGM_TYPE_32BIT,
587	PGM_GST_NAME_32BIT(GetPage),
588	PGM_GST_NAME_32BIT(ModifyPage),
589	PGM_GST_NAME_32BIT(Enter),
590	PGM_GST_NAME_32BIT(Exit),
591	#ifdef IN_RING3
592	PGM_GST_NAME_32BIT(Relocate),
593	#endif
594	},
595	{
596	PGM_TYPE_PAE,
597	PGM_GST_NAME_PAE(GetPage),
598	PGM_GST_NAME_PAE(ModifyPage),
599	PGM_GST_NAME_PAE(Enter),
600	PGM_GST_NAME_PAE(Exit),
601	#ifdef IN_RING3
602	PGM_GST_NAME_PAE(Relocate),
603	#endif
604	},
605	#ifdef VBOX_WITH_64_BITS_GUESTS
606	{
607	PGM_TYPE_AMD64,
608	PGM_GST_NAME_AMD64(GetPage),
609	PGM_GST_NAME_AMD64(ModifyPage),
610	PGM_GST_NAME_AMD64(Enter),
611	PGM_GST_NAME_AMD64(Exit),
612	# ifdef IN_RING3
613	PGM_GST_NAME_AMD64(Relocate),
614	# endif
615	},
616	#endif
617	};
618
619
620	/**
621	* The shadow mode data array.
622	*/
623	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
624	{
625	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
626	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
627	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
628	{
629	PGM_TYPE_32BIT,
630	PGM_SHW_NAME_32BIT(GetPage),
631	PGM_SHW_NAME_32BIT(ModifyPage),
632	PGM_SHW_NAME_32BIT(Enter),
633	PGM_SHW_NAME_32BIT(Exit),
634	#ifdef IN_RING3
635	PGM_SHW_NAME_32BIT(Relocate),
636	#endif
637	},
638	{
639	PGM_TYPE_PAE,
640	PGM_SHW_NAME_PAE(GetPage),
641	PGM_SHW_NAME_PAE(ModifyPage),
642	PGM_SHW_NAME_PAE(Enter),
643	PGM_SHW_NAME_PAE(Exit),
644	#ifdef IN_RING3
645	PGM_SHW_NAME_PAE(Relocate),
646	#endif
647	},
648	{
649	PGM_TYPE_AMD64,
650	PGM_SHW_NAME_AMD64(GetPage),
651	PGM_SHW_NAME_AMD64(ModifyPage),
652	PGM_SHW_NAME_AMD64(Enter),
653	PGM_SHW_NAME_AMD64(Exit),
654	#ifdef IN_RING3
655	PGM_SHW_NAME_AMD64(Relocate),
656	#endif
657	},
658	{
659	PGM_TYPE_NESTED_32BIT,
660	PGM_SHW_NAME_NESTED_32BIT(GetPage),
661	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
662	PGM_SHW_NAME_NESTED_32BIT(Enter),
663	PGM_SHW_NAME_NESTED_32BIT(Exit),
664	#ifdef IN_RING3
665	PGM_SHW_NAME_NESTED_32BIT(Relocate),
666	#endif
667	},
668	{
669	PGM_TYPE_NESTED_PAE,
670	PGM_SHW_NAME_NESTED_PAE(GetPage),
671	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
672	PGM_SHW_NAME_NESTED_PAE(Enter),
673	PGM_SHW_NAME_NESTED_PAE(Exit),
674	#ifdef IN_RING3
675	PGM_SHW_NAME_NESTED_PAE(Relocate),
676	#endif
677	},
678	{
679	PGM_TYPE_NESTED_AMD64,
680	PGM_SHW_NAME_NESTED_AMD64(GetPage),
681	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
682	PGM_SHW_NAME_NESTED_AMD64(Enter),
683	PGM_SHW_NAME_NESTED_AMD64(Exit),
684	#ifdef IN_RING3
685	PGM_SHW_NAME_NESTED_AMD64(Relocate),
686	#endif
687	},
688	{
689	PGM_TYPE_EPT,
690	PGM_SHW_NAME_EPT(GetPage),
691	PGM_SHW_NAME_EPT(ModifyPage),
692	PGM_SHW_NAME_EPT(Enter),
693	PGM_SHW_NAME_EPT(Exit),
694	#ifdef IN_RING3
695	PGM_SHW_NAME_EPT(Relocate),
696	#endif
697	},
698	{
699	PGM_TYPE_NONE,
700	PGM_SHW_NAME_NONE(GetPage),
701	PGM_SHW_NAME_NONE(ModifyPage),
702	PGM_SHW_NAME_NONE(Enter),
703	PGM_SHW_NAME_NONE(Exit),
704	#ifdef IN_RING3
705	PGM_SHW_NAME_NONE(Relocate),
706	#endif
707	},
708	};
709
710
711	/**
712	* The guest+shadow mode data array.
713	*/
714	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
715	{
716	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
717	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
718	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
719	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler) }
720
721	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
722	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
723	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
724	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(AssertCR3) }
725
726	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
727	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
728	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
729	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
730
731	#elif defined(IN_RING3) && defined(VBOX_STRICT)
732	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
733	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
734	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
735
736	#else
737	# error "Misconfig."
738	#endif
739
740	/* 32-bit shadow paging mode: */
741	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
742	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
743	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
744	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
745	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
746	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
747	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
748	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
749	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
750	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
751	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
752
753	/* PAE shadow paging mode: */
754	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
755	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
756	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
757	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
758	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
759	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
760	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
761	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
762	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
763	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
764	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
765
766	/* AMD64 shadow paging mode: */
767	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
768	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
769	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
770	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
771	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
772	#ifdef VBOX_WITH_64_BITS_GUESTS
773	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
774	#else
775	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
776	#endif
777	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
782
783	/* 32-bit nested paging mode: */
784	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
785	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
789	#ifdef VBOX_WITH_64_BITS_GUESTS
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
791	#else
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
793	#endif
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
797	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
798	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
799
800	/* PAE nested paging mode: */
801	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
802	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
803	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
805	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
806	#ifdef VBOX_WITH_64_BITS_GUESTS
807	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
808	#else
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
810	#endif
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
813	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
814	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
815	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
816
817	/* AMD64 nested paging mode: */
818	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
820	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
821	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
822	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
823	#ifdef VBOX_WITH_64_BITS_GUESTS
824	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
825	#else
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
827	#endif
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
830	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
831	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
832	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
833
834	/* EPT nested paging mode: */
835	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
837	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
838	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
839	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
840	#ifdef VBOX_WITH_64_BITS_GUESTS
841	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
842	#else
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
844	#endif
845	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
846	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
847	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
848	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
849	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
850
851	/* NONE / NEM: */
852	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
854	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
855	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
856	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
857	#ifdef VBOX_WITH_64_BITS_GUESTS
858	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
859	#else
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
861	#endif
862	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
863	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
864	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
865	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
866	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
867
868
869	#undef PGMMODEDATABTH_ENTRY
870	#undef PGMMODEDATABTH_NULL_ENTRY
871	};
872
873
874	#ifdef IN_RING0
875	/**
876	* #PF Handler.
877	*
878	* @returns VBox status code (appropriate for trap handling and GC return).
879	* @param pVCpu The cross context virtual CPU structure.
880	* @param uErr The trap error code.
881	* @param pRegFrame Trap register frame.
882	* @param pvFault The fault address.
883	*/
884	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
885	{
886	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
887
888	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
889	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
890	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
891
892
893	# ifdef VBOX_WITH_STATISTICS
894	/*
895	* Error code stats.
896	*/
897	if (uErr & X86_TRAP_PF_US)
898	{
899	if (!(uErr & X86_TRAP_PF_P))
900	{
901	if (uErr & X86_TRAP_PF_RW)
902	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
903	else
904	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
905	}
906	else if (uErr & X86_TRAP_PF_RW)
907	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
908	else if (uErr & X86_TRAP_PF_RSVD)
909	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
910	else if (uErr & X86_TRAP_PF_ID)
911	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
912	else
913	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
914	}
915	else
916	{ /* Supervisor */
917	if (!(uErr & X86_TRAP_PF_P))
918	{
919	if (uErr & X86_TRAP_PF_RW)
920	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
921	else
922	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
923	}
924	else if (uErr & X86_TRAP_PF_RW)
925	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
926	else if (uErr & X86_TRAP_PF_ID)
927	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
928	else if (uErr & X86_TRAP_PF_RSVD)
929	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
930	}
931	# endif /* VBOX_WITH_STATISTICS */
932
933	/*
934	* Call the worker.
935	*/
936	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
937	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
938	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
939	bool fLockTaken = false;
940	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
941	if (fLockTaken)
942	{
943	PGM_LOCK_ASSERT_OWNER(pVM);
944	PGM_UNLOCK(pVM);
945	}
946	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
947
948	/*
949	* Return code tweaks.
950	*/
951	if (rc != VINF_SUCCESS)
952	{
953	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
954	rc = VINF_SUCCESS;
955
956	/* Note: hack alert for difficult to reproduce problem. */
957	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
958	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
959	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
960	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
961	{
962	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
963	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
964	rc = VINF_SUCCESS;
965	}
966	}
967
968	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
969	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
970	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
971	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
972	return rc;
973	}
974	#endif /* IN_RING0 */
975
976
977	/**
978	* Prefetch a page
979	*
980	* Typically used to sync commonly used pages before entering raw mode
981	* after a CR3 reload.
982	*
983	* @returns VBox status code suitable for scheduling.
984	* @retval VINF_SUCCESS on success.
985	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
986	* @param pVCpu The cross context virtual CPU structure.
987	* @param GCPtrPage Page to invalidate.
988	*/
989	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
990	{
991	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
992
993	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
994	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
995	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
996	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
997
998	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
999	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1000	return rc;
1001	}
1002
1003
1004	#ifndef PGM_WITHOUT_MAPPINGS
1005	/**
1006	* Gets the mapping corresponding to the specified address (if any).
1007	*
1008	* @returns Pointer to the mapping.
1009	* @returns NULL if not
1010	*
1011	* @param pVM The cross context VM structure.
1012	* @param GCPtr The guest context pointer.
1013	*/
1014	PPGMMAPPING pgmGetMapping(PVM pVM, RTGCPTR GCPtr)
1015	{
1016	PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
1017	while (pMapping)
1018	{
1019	if ((uintptr_t)GCPtr < (uintptr_t)pMapping->GCPtr)
1020	break;
1021	if ((uintptr_t)GCPtr - (uintptr_t)pMapping->GCPtr < pMapping->cb)
1022	return pMapping;
1023	pMapping = pMapping->CTX_SUFF(pNext);
1024	}
1025	return NULL;
1026	}
1027	#endif
1028
1029
1030	/**
1031	* Verifies a range of pages for read or write access
1032	*
1033	* Only checks the guest's page tables
1034	*
1035	* @returns VBox status code.
1036	* @param pVCpu The cross context virtual CPU structure.
1037	* @param Addr Guest virtual address to check
1038	* @param cbSize Access size
1039	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1040	* @remarks Current not in use.
1041	*/
1042	VMMDECL(int) PGMIsValidAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1043	{
1044	/*
1045	* Validate input.
1046	*/
1047	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
1048	{
1049	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
1050	return VERR_INVALID_PARAMETER;
1051	}
1052
1053	uint64_t fPage;
1054	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
1055	if (RT_FAILURE(rc))
1056	{
1057	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
1058	return VINF_EM_RAW_GUEST_TRAP;
1059	}
1060
1061	/*
1062	* Check if the access would cause a page fault
1063	*
1064	* Note that hypervisor page directories are not present in the guest's tables, so this check
1065	* is sufficient.
1066	*/
1067	bool fWrite = !!(fAccess & X86_PTE_RW);
1068	bool fUser = !!(fAccess & X86_PTE_US);
1069	if ( !(fPage & X86_PTE_P)
1070	\|\| (fWrite && !(fPage & X86_PTE_RW))
1071	\|\| (fUser && !(fPage & X86_PTE_US)) )
1072	{
1073	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
1074	return VINF_EM_RAW_GUEST_TRAP;
1075	}
1076	if ( RT_SUCCESS(rc)
1077	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
1078	return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
1079	return rc;
1080	}
1081
1082
1083	/**
1084	* Verifies a range of pages for read or write access
1085	*
1086	* Supports handling of pages marked for dirty bit tracking and CSAM
1087	*
1088	* @returns VBox status code.
1089	* @param pVCpu The cross context virtual CPU structure.
1090	* @param Addr Guest virtual address to check
1091	* @param cbSize Access size
1092	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1093	*/
1094	VMMDECL(int) PGMVerifyAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1095	{
1096	PVM pVM = pVCpu->CTX_SUFF(pVM);
1097
1098	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
1099
1100	/*
1101	* Get going.
1102	*/
1103	uint64_t fPageGst;
1104	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
1105	if (RT_FAILURE(rc))
1106	{
1107	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
1108	return VINF_EM_RAW_GUEST_TRAP;
1109	}
1110
1111	/*
1112	* Check if the access would cause a page fault
1113	*
1114	* Note that hypervisor page directories are not present in the guest's tables, so this check
1115	* is sufficient.
1116	*/
1117	const bool fWrite = !!(fAccess & X86_PTE_RW);
1118	const bool fUser = !!(fAccess & X86_PTE_US);
1119	if ( !(fPageGst & X86_PTE_P)
1120	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
1121	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
1122	{
1123	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
1124	return VINF_EM_RAW_GUEST_TRAP;
1125	}
1126
1127	if (!pVM->pgm.s.fNestedPaging)
1128	{
1129	/*
1130	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
1131	*/
1132	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
1133	if ( rc == VERR_PAGE_NOT_PRESENT
1134	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
1135	{
1136	/*
1137	* Page is not present in our page tables.
1138	* Try to sync it!
1139	*/
1140	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
1141	uint32_t const uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
1142	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1143	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1144	AssertReturn(g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
1145	rc = g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage(pVCpu, Addr, fPageGst, uErr);
1146	if (rc != VINF_SUCCESS)
1147	return rc;
1148	}
1149	else
1150	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
1151	}
1152
1153	#if 0 /* def VBOX_STRICT; triggers too often now */
1154	/*
1155	* This check is a bit paranoid, but useful.
1156	*/
1157	/* Note! This will assert when writing to monitored pages (a bit annoying actually). */
1158	uint64_t fPageShw;
1159	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
1160	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
1161	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
1162	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
1163	{
1164	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
1165	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
1166	return VINF_EM_RAW_GUEST_TRAP;
1167	}
1168	#endif
1169
1170	if ( RT_SUCCESS(rc)
1171	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
1172	\|\| Addr + cbSize < Addr))
1173	{
1174	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
1175	for (;;)
1176	{
1177	Addr += PAGE_SIZE;
1178	if (cbSize > PAGE_SIZE)
1179	cbSize -= PAGE_SIZE;
1180	else
1181	cbSize = 1;
1182	rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
1183	if (rc != VINF_SUCCESS)
1184	break;
1185	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
1186	break;
1187	}
1188	}
1189	return rc;
1190	}
1191
1192
1193	/**
1194	* Emulation of the invlpg instruction (HC only actually).
1195	*
1196	* @returns Strict VBox status code, special care required.
1197	* @retval VINF_PGM_SYNC_CR3 - handled.
1198	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1199	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1200	*
1201	* @param pVCpu The cross context virtual CPU structure.
1202	* @param GCPtrPage Page to invalidate.
1203	*
1204	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1205	* safe, but there could be edge cases!
1206	*
1207	* @todo Flush page or page directory only if necessary!
1208	* @todo VBOXSTRICTRC
1209	*/
1210	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1211	{
1212	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1213	int rc;
1214	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1215
1216	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1217
1218	/*
1219	* Call paging mode specific worker.
1220	*/
1221	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1222	PGM_LOCK_VOID(pVM);
1223
1224	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1225	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1226	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1227	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1228
1229	PGM_UNLOCK(pVM);
1230	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1231
1232	#ifdef IN_RING3
1233	/*
1234	* Check if we have a pending update of the CR3 monitoring.
1235	*/
1236	if ( RT_SUCCESS(rc)
1237	&& (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3))
1238	{
1239	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
1240	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
1241	}
1242	#endif /* IN_RING3 */
1243
1244	/* Ignore all irrelevant error codes. */
1245	if ( rc == VERR_PAGE_NOT_PRESENT
1246	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1247	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1248	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1249	rc = VINF_SUCCESS;
1250
1251	return rc;
1252	}
1253
1254
1255	/**
1256	* Executes an instruction using the interpreter.
1257	*
1258	* @returns VBox status code (appropriate for trap handling and GC return).
1259	* @param pVM The cross context VM structure.
1260	* @param pVCpu The cross context virtual CPU structure.
1261	* @param pRegFrame Register frame.
1262	* @param pvFault Fault address.
1263	*/
1264	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1265	{
1266	NOREF(pVM);
1267	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1268	if (rc == VERR_EM_INTERPRETER)
1269	rc = VINF_EM_RAW_EMULATE_INSTR;
1270	if (rc != VINF_SUCCESS)
1271	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1272	return rc;
1273	}
1274
1275
1276	/**
1277	* Gets effective page information (from the VMM page directory).
1278	*
1279	* @returns VBox status code.
1280	* @param pVCpu The cross context virtual CPU structure.
1281	* @param GCPtr Guest Context virtual address of the page.
1282	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1283	* @param pHCPhys Where to store the HC physical address of the page.
1284	* This is page aligned.
1285	* @remark You should use PGMMapGetPage() for pages in a mapping.
1286	*/
1287	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1288	{
1289	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1290	PGM_LOCK_VOID(pVM);
1291
1292	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1293	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1294	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1295	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1296
1297	PGM_UNLOCK(pVM);
1298	return rc;
1299	}
1300
1301
1302	/**
1303	* Modify page flags for a range of pages in the shadow context.
1304	*
1305	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1306	*
1307	* @returns VBox status code.
1308	* @param pVCpu The cross context virtual CPU structure.
1309	* @param GCPtr Virtual address of the first page in the range.
1310	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1311	* @param fMask The AND mask - page flags X86_PTE_*.
1312	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1313	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1314	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1315	*/
1316	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1317	{
1318	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1319	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1320
1321	GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
1322
1323	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1324	PGM_LOCK_VOID(pVM);
1325
1326	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1327	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1328	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1329	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
1330
1331	PGM_UNLOCK(pVM);
1332	return rc;
1333	}
1334
1335
1336	/**
1337	* Changing the page flags for a single page in the shadow page tables so as to
1338	* make it read-only.
1339	*
1340	* @returns VBox status code.
1341	* @param pVCpu The cross context virtual CPU structure.
1342	* @param GCPtr Virtual address of the first page in the range.
1343	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1344	*/
1345	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1346	{
1347	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1348	}
1349
1350
1351	/**
1352	* Changing the page flags for a single page in the shadow page tables so as to
1353	* make it writable.
1354	*
1355	* The call must know with 101% certainty that the guest page tables maps this
1356	* as writable too. This function will deal shared, zero and write monitored
1357	* pages.
1358	*
1359	* @returns VBox status code.
1360	* @param pVCpu The cross context virtual CPU structure.
1361	* @param GCPtr Virtual address of the first page in the range.
1362	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1363	*/
1364	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1365	{
1366	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1367	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1368	return VINF_SUCCESS;
1369	}
1370
1371
1372	/**
1373	* Changing the page flags for a single page in the shadow page tables so as to
1374	* make it not present.
1375	*
1376	* @returns VBox status code.
1377	* @param pVCpu The cross context virtual CPU structure.
1378	* @param GCPtr Virtual address of the first page in the range.
1379	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1380	*/
1381	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1382	{
1383	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1384	}
1385
1386
1387	/**
1388	* Changing the page flags for a single page in the shadow page tables so as to
1389	* make it supervisor and writable.
1390	*
1391	* This if for dealing with CR0.WP=0 and readonly user pages.
1392	*
1393	* @returns VBox status code.
1394	* @param pVCpu The cross context virtual CPU structure.
1395	* @param GCPtr Virtual address of the first page in the range.
1396	* @param fBigPage Whether or not this is a big page. If it is, we have to
1397	* change the shadow PDE as well. If it isn't, the caller
1398	* has checked that the shadow PDE doesn't need changing.
1399	* We ASSUME 4KB pages backing the big page here!
1400	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1401	*/
1402	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1403	{
1404	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1405	if (rc == VINF_SUCCESS && fBigPage)
1406	{
1407	/* this is a bit ugly... */
1408	switch (pVCpu->pgm.s.enmShadowMode)
1409	{
1410	case PGMMODE_32_BIT:
1411	{
1412	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1413	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1414	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1415	pPde->u \|= X86_PDE_RW;
1416	Log(("-> PDE=%#llx (32)\n", pPde->u));
1417	break;
1418	}
1419	case PGMMODE_PAE:
1420	case PGMMODE_PAE_NX:
1421	{
1422	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1423	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1424	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1425	pPde->u \|= X86_PDE_RW;
1426	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1427	break;
1428	}
1429	default:
1430	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1431	}
1432	}
1433	return rc;
1434	}
1435
1436
1437	/**
1438	* Gets the shadow page directory for the specified address, PAE.
1439	*
1440	* @returns Pointer to the shadow PD.
1441	* @param pVCpu The cross context virtual CPU structure.
1442	* @param GCPtr The address.
1443	* @param uGstPdpe Guest PDPT entry. Valid.
1444	* @param ppPD Receives address of page directory
1445	*/
1446	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1447	{
1448	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1449	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1450	PPGMPOOLPAGE pShwPage;
1451	int rc;
1452	PGM_LOCK_ASSERT_OWNER(pVM);
1453
1454
1455	/* Allocate page directory if not present. */
1456	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1457	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1458	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1459	X86PGPAEUINT const uPdpe = pPdpe->u;
1460	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1461	{
1462	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1463	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1464	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1465
1466	pgmPoolCacheUsed(pPool, pShwPage);
1467
1468	/* Update the entry if necessary. */
1469	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1470	if (uPdpeNew == uPdpe)
1471	{ /* likely */ }
1472	else
1473	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1474	}
1475	else
1476	{
1477	RTGCPTR64 GCPdPt;
1478	PGMPOOLKIND enmKind;
1479	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1480	{
1481	/* AMD-V nested paging or real/protected mode without paging. */
1482	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1483	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1484	}
1485	else
1486	{
1487	if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1488	{
1489	if (!(uGstPdpe & X86_PDPE_P))
1490	{
1491	/* PD not present; guest must reload CR3 to change it.
1492	* No need to monitor anything in this case. */
1493	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1494	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1495	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1496	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1497	}
1498	else
1499	{
1500	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1501	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1502	}
1503	}
1504	else
1505	{
1506	GCPdPt = CPUMGetGuestCR3(pVCpu);
1507	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1508	}
1509	}
1510
1511	/* Create a reference back to the PDPT by using the index in its shadow page. */
1512	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1513	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1514	&pShwPage);
1515	AssertRCReturn(rc, rc);
1516
1517	/* Hook it up. */
1518	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1519	}
1520	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1521
1522	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1523	return VINF_SUCCESS;
1524	}
1525
1526
1527	/**
1528	* Gets the pointer to the shadow page directory entry for an address, PAE.
1529	*
1530	* @returns Pointer to the PDE.
1531	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1532	* @param GCPtr The address.
1533	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1534	*/
1535	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1536	{
1537	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1538	PGM_LOCK_ASSERT_OWNER(pVM);
1539
1540	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1541	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1542	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1543	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1544	if (!(uPdpe & X86_PDPE_P))
1545	{
1546	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1547	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1548	}
1549	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1550
1551	/* Fetch the pgm pool shadow descriptor. */
1552	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1553	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1554
1555	*ppShwPde = pShwPde;
1556	return VINF_SUCCESS;
1557	}
1558
1559
1560	/**
1561	* Syncs the SHADOW page directory pointer for the specified address.
1562	*
1563	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1564	*
1565	* The caller is responsible for making sure the guest has a valid PD before
1566	* calling this function.
1567	*
1568	* @returns VBox status code.
1569	* @param pVCpu The cross context virtual CPU structure.
1570	* @param GCPtr The address.
1571	* @param uGstPml4e Guest PML4 entry (valid).
1572	* @param uGstPdpe Guest PDPT entry (valid).
1573	* @param ppPD Receives address of page directory
1574	*/
1575	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1576	{
1577	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1578	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1579	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1580	int rc;
1581
1582	PGM_LOCK_ASSERT_OWNER(pVM);
1583
1584	/*
1585	* PML4.
1586	*/
1587	PPGMPOOLPAGE pShwPage;
1588	{
1589	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1590	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1591	X86PGPAEUINT const uPml4e = pPml4e->u;
1592
1593	/* Allocate page directory pointer table if not present. */
1594	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1595	{
1596	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1597	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1598
1599	pgmPoolCacheUsed(pPool, pShwPage);
1600
1601	/* Update the entry if needed. */
1602	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1603	\| (uPml4e & PGM_PML4_FLAGS);
1604	if (uPml4e == uPml4eNew)
1605	{ /* likely */ }
1606	else
1607	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1608	}
1609	else
1610	{
1611	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1612
1613	RTGCPTR64 GCPml4;
1614	PGMPOOLKIND enmKind;
1615	if (fNestedPagingOrNoGstPaging)
1616	{
1617	/* AMD-V nested paging or real/protected mode without paging */
1618	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1619	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1620	}
1621	else
1622	{
1623	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1624	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1625	}
1626
1627	/* Create a reference back to the PDPT by using the index in its shadow page. */
1628	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1629	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1630	&pShwPage);
1631	AssertRCReturn(rc, rc);
1632
1633	/* Hook it up. */
1634	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1635	\| (uPml4e & PGM_PML4_FLAGS));
1636	}
1637	}
1638
1639	/*
1640	* PDPT.
1641	*/
1642	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1643	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1644	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1645	X86PGPAEUINT const uPdpe = pPdpe->u;
1646
1647	/* Allocate page directory if not present. */
1648	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1649	{
1650	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1651	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1652
1653	pgmPoolCacheUsed(pPool, pShwPage);
1654
1655	/* Update the entry if needed. */
1656	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1657	\| (uPdpe & PGM_PDPT_FLAGS);
1658	if (uPdpe == uPdpeNew)
1659	{ /* likely */ }
1660	else
1661	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1662	}
1663	else
1664	{
1665	RTGCPTR64 GCPdPt;
1666	PGMPOOLKIND enmKind;
1667	if (fNestedPagingOrNoGstPaging)
1668	{
1669	/* AMD-V nested paging or real/protected mode without paging */
1670	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1671	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1672	}
1673	else
1674	{
1675	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1676	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1677	}
1678
1679	/* Create a reference back to the PDPT by using the index in its shadow page. */
1680	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1681	pShwPage->idx, iPdPt, false /fLockPage/,
1682	&pShwPage);
1683	AssertRCReturn(rc, rc);
1684
1685	/* Hook it up. */
1686	ASMAtomicWriteU64(&pPdpe->u,
1687	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1688	}
1689
1690	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1691	return VINF_SUCCESS;
1692	}
1693
1694
1695	/**
1696	* Gets the SHADOW page directory pointer for the specified address (long mode).
1697	*
1698	* @returns VBox status code.
1699	* @param pVCpu The cross context virtual CPU structure.
1700	* @param GCPtr The address.
1701	* @param ppPml4e Receives the address of the page map level 4 entry.
1702	* @param ppPdpt Receives the address of the page directory pointer table.
1703	* @param ppPD Receives the address of the page directory.
1704	*/
1705	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1706	{
1707	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1708	PGM_LOCK_ASSERT_OWNER(pVM);
1709
1710	/*
1711	* PML4
1712	*/
1713	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1714	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1715	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1716	if (ppPml4e)
1717	*ppPml4e = (PX86PML4E)pPml4e;
1718	X86PGPAEUINT const uPml4e = pPml4e->u;
1719	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1720	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1721	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1722
1723	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1724	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1725	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1726
1727	/*
1728	* PDPT
1729	*/
1730	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1731	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1732	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1733	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1734	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1735
1736	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1737	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1738
1739	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1740	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1741	return VINF_SUCCESS;
1742	}
1743
1744
1745	/**
1746	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1747	* backing pages in case the PDPT or PML4 entry is missing.
1748	*
1749	* @returns VBox status code.
1750	* @param pVCpu The cross context virtual CPU structure.
1751	* @param GCPtr The address.
1752	* @param ppPdpt Receives address of pdpt
1753	* @param ppPD Receives address of page directory
1754	*/
1755	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1756	{
1757	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1758	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1759	int rc;
1760
1761	Assert(pVM->pgm.s.fNestedPaging);
1762	PGM_LOCK_ASSERT_OWNER(pVM);
1763
1764	/*
1765	* PML4 level.
1766	*/
1767
1768	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1769	Assert(pPml4);
1770
1771	/* Allocate page directory pointer table if not present. */
1772	PPGMPOOLPAGE pShwPage;
1773	{
1774	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1775	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1776	EPTPML4E Pml4e;
1777	Pml4e.u = pPml4e->u;
1778	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1779	{
1780	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1781
1782	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1783	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1784	&pShwPage);
1785	AssertRCReturn(rc, rc);
1786
1787	/* Hook up the new PDPT now. */
1788	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1789	}
1790	else
1791	{
1792	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1793	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1794
1795	pgmPoolCacheUsed(pPool, pShwPage);
1796
1797	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1798	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1799	{ }
1800	else
1801	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1802	}
1803	}
1804
1805	/*
1806	* PDPT level.
1807	*/
1808	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1809	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1810	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1811
1812	if (ppPdpt)
1813	*ppPdpt = pPdpt;
1814
1815	/* Allocate page directory if not present. */
1816	EPTPDPTE Pdpe;
1817	Pdpe.u = pPdpe->u;
1818	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1819	{
1820	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1821	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1822	pShwPage->idx, iPdPt, false /fLockPage/,
1823	&pShwPage);
1824	AssertRCReturn(rc, rc);
1825
1826	/* Hook up the new PD now. */
1827	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1828	}
1829	else
1830	{
1831	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1832	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1833
1834	pgmPoolCacheUsed(pPool, pShwPage);
1835
1836	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1837	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1838	{ }
1839	else
1840	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1841	}
1842
1843	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1844	return VINF_SUCCESS;
1845	}
1846
1847
1848	#ifdef IN_RING0
1849	/**
1850	* Synchronizes a range of nested page table entries.
1851	*
1852	* The caller must own the PGM lock.
1853	*
1854	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1855	* @param GCPhys Where to start.
1856	* @param cPages How many pages which entries should be synced.
1857	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1858	* host paging mode for AMD-V).
1859	*/
1860	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1861	{
1862	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1863
1864	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1865	int rc;
1866	switch (enmShwPagingMode)
1867	{
1868	case PGMMODE_32_BIT:
1869	{
1870	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1871	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1872	break;
1873	}
1874
1875	case PGMMODE_PAE:
1876	case PGMMODE_PAE_NX:
1877	{
1878	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1879	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1880	break;
1881	}
1882
1883	case PGMMODE_AMD64:
1884	case PGMMODE_AMD64_NX:
1885	{
1886	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1887	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1888	break;
1889	}
1890
1891	case PGMMODE_EPT:
1892	{
1893	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1894	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1895	break;
1896	}
1897
1898	default:
1899	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1900	}
1901	return rc;
1902	}
1903	#endif /* IN_RING0 */
1904
1905
1906	/**
1907	* Gets effective Guest OS page information.
1908	*
1909	* When GCPtr is in a big page, the function will return as if it was a normal
1910	* 4KB page. If the need for distinguishing between big and normal page becomes
1911	* necessary at a later point, a PGMGstGetPage() will be created for that
1912	* purpose.
1913	*
1914	* @returns VBox status code.
1915	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1916	* @param GCPtr Guest Context virtual address of the page.
1917	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1918	* @param pGCPhys Where to store the GC physical address of the page.
1919	* This is page aligned. The fact that the
1920	*/
1921	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1922	{
1923	VMCPU_ASSERT_EMT(pVCpu);
1924	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1925	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1926	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1927	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
1928	}
1929
1930
1931	/**
1932	* Performs a guest page table walk.
1933	*
1934	* The guest should be in paged protect mode or long mode when making a call to
1935	* this function.
1936	*
1937	* @returns VBox status code.
1938	* @retval VINF_SUCCESS on success.
1939	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1940	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1941	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1942	*
1943	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1944	* @param GCPtr The guest virtual address to walk by.
1945	* @param pWalk Where to return the walk result. This is valid for some
1946	* error codes as well.
1947	*/
1948	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1949	{
1950	VMCPU_ASSERT_EMT(pVCpu);
1951	switch (pVCpu->pgm.s.enmGuestMode)
1952	{
1953	case PGMMODE_32_BIT:
1954	pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1955	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
1956
1957	case PGMMODE_PAE:
1958	case PGMMODE_PAE_NX:
1959	pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1960	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
1961
1962	case PGMMODE_AMD64:
1963	case PGMMODE_AMD64_NX:
1964	pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1965	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
1966
1967	case PGMMODE_REAL:
1968	case PGMMODE_PROTECTED:
1969	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1970	return VERR_PGM_NOT_USED_IN_MODE;
1971
1972	case PGMMODE_NESTED_32BIT:
1973	case PGMMODE_NESTED_PAE:
1974	case PGMMODE_NESTED_AMD64:
1975	case PGMMODE_EPT:
1976	default:
1977	AssertFailed();
1978	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1979	return VERR_PGM_NOT_USED_IN_MODE;
1980	}
1981	}
1982
1983
1984	/**
1985	* Tries to continue the previous walk.
1986	*
1987	* @note Requires the caller to hold the PGM lock from the first
1988	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1989	* we cannot use the pointers.
1990	*
1991	* @returns VBox status code.
1992	* @retval VINF_SUCCESS on success.
1993	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1994	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1995	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1996	*
1997	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1998	* @param GCPtr The guest virtual address to walk by.
1999	* @param pWalk Pointer to the previous walk result and where to return
2000	* the result of this walk. This is valid for some error
2001	* codes as well.
2002	*/
2003	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
2004	{
2005	/*
2006	* We can only handle successfully walks.
2007	* We also limit ourselves to the next page.
2008	*/
2009	if ( pWalk->u.Core.fSucceeded
2010	&& GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
2011	{
2012	Assert(pWalk->u.Core.uLevel == 0);
2013	if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
2014	{
2015	/*
2016	* AMD64
2017	*/
2018	if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
2019	{
2020	/*
2021	* We fall back to full walk if the PDE table changes, if any
2022	* reserved bits are set, or if the effective page access changes.
2023	*/
2024	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
2025	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
2026	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
2027	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
2028
2029	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
2030	{
2031	if (pWalk->u.Amd64.pPte)
2032	{
2033	X86PTEPAE Pte;
2034	Pte.u = pWalk->u.Amd64.pPte[1].u;
2035	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2036	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2037	{
2038
2039	pWalk->u.Core.GCPtr = GCPtr;
2040	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2041	pWalk->u.Amd64.Pte.u = Pte.u;
2042	pWalk->u.Amd64.pPte++;
2043	return VINF_SUCCESS;
2044	}
2045	}
2046	}
2047	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
2048	{
2049	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2050	if (pWalk->u.Amd64.pPde)
2051	{
2052	X86PDEPAE Pde;
2053	Pde.u = pWalk->u.Amd64.pPde[1].u;
2054	if ( (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
2055	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2056	{
2057	/* Get the new PTE and check out the first entry. */
2058	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2059	&pWalk->u.Amd64.pPt);
2060	if (RT_SUCCESS(rc))
2061	{
2062	pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
2063	X86PTEPAE Pte;
2064	Pte.u = pWalk->u.Amd64.pPte->u;
2065	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2066	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2067	{
2068	pWalk->u.Core.GCPtr = GCPtr;
2069	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2070	pWalk->u.Amd64.Pte.u = Pte.u;
2071	pWalk->u.Amd64.Pde.u = Pde.u;
2072	pWalk->u.Amd64.pPde++;
2073	return VINF_SUCCESS;
2074	}
2075	}
2076	}
2077	}
2078	}
2079	}
2080	else if (!pWalk->u.Core.fGigantPage)
2081	{
2082	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
2083	{
2084	pWalk->u.Core.GCPtr = GCPtr;
2085	pWalk->u.Core.GCPhys += PAGE_SIZE;
2086	return VINF_SUCCESS;
2087	}
2088	}
2089	else
2090	{
2091	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
2092	{
2093	pWalk->u.Core.GCPtr = GCPtr;
2094	pWalk->u.Core.GCPhys += PAGE_SIZE;
2095	return VINF_SUCCESS;
2096	}
2097	}
2098	}
2099	}
2100	/* Case we don't handle. Do full walk. */
2101	return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
2102	}
2103
2104
2105	/**
2106	* Checks if the page is present.
2107	*
2108	* @returns true if the page is present.
2109	* @returns false if the page is not present.
2110	* @param pVCpu The cross context virtual CPU structure.
2111	* @param GCPtr Address within the page.
2112	*/
2113	VMMDECL(bool) PGMGstIsPagePresent(PVMCPUCC pVCpu, RTGCPTR GCPtr)
2114	{
2115	VMCPU_ASSERT_EMT(pVCpu);
2116	int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
2117	return RT_SUCCESS(rc);
2118	}
2119
2120
2121	/**
2122	* Sets (replaces) the page flags for a range of pages in the guest's tables.
2123	*
2124	* @returns VBox status code.
2125	* @param pVCpu The cross context virtual CPU structure.
2126	* @param GCPtr The address of the first page.
2127	* @param cb The size of the range in bytes.
2128	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
2129	*/
2130	VMMDECL(int) PGMGstSetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
2131	{
2132	VMCPU_ASSERT_EMT(pVCpu);
2133	return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
2134	}
2135
2136
2137	/**
2138	* Modify page flags for a range of pages in the guest's tables
2139	*
2140	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2141	*
2142	* @returns VBox status code.
2143	* @param pVCpu The cross context virtual CPU structure.
2144	* @param GCPtr Virtual address of the first page in the range.
2145	* @param cb Size (in bytes) of the range to apply the modification to.
2146	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2147	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2148	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2149	*/
2150	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2151	{
2152	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2153	VMCPU_ASSERT_EMT(pVCpu);
2154
2155	/*
2156	* Validate input.
2157	*/
2158	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2159	Assert(cb);
2160
2161	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2162
2163	/*
2164	* Adjust input.
2165	*/
2166	cb += GCPtr & PAGE_OFFSET_MASK;
2167	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
2168	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
2169
2170	/*
2171	* Call worker.
2172	*/
2173	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2174	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2175	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2176	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2177
2178	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2179	return rc;
2180	}
2181
2182
2183	/**
2184	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2185	*
2186	* @returns @c true if the PDPE is valid, @c false otherwise.
2187	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2188	* @param paPaePdpes The PAE PDPEs to validate.
2189	*
2190	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2191	*/
2192	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2193	{
2194	Assert(paPaePdpes);
2195	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2196	{
2197	X86PDPE const PaePdpe = paPaePdpes[i];
2198	if ( !(PaePdpe.u & X86_PDPE_P)
2199	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2200	{ /* likely */ }
2201	else
2202	return false;
2203	}
2204	return true;
2205	}
2206
2207
2208	/**
2209	* Performs the lazy mapping of the 32-bit guest PD.
2210	*
2211	* @returns VBox status code.
2212	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2213	* @param ppPd Where to return the pointer to the mapping. This is
2214	* always set.
2215	*/
2216	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2217	{
2218	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2219	PGM_LOCK_VOID(pVM);
2220
2221	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2222
2223	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2224	PPGMPAGE pPage;
2225	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2226	if (RT_SUCCESS(rc))
2227	{
2228	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2229	if (RT_SUCCESS(rc))
2230	{
2231	# ifdef IN_RING3
2232	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2233	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2234	# else
2235	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2236	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2237	# endif
2238	PGM_UNLOCK(pVM);
2239	return VINF_SUCCESS;
2240	}
2241	AssertRC(rc);
2242	}
2243	PGM_UNLOCK(pVM);
2244
2245	*ppPd = NULL;
2246	return rc;
2247	}
2248
2249
2250	/**
2251	* Performs the lazy mapping of the PAE guest PDPT.
2252	*
2253	* @returns VBox status code.
2254	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2255	* @param ppPdpt Where to return the pointer to the mapping. This is
2256	* always set.
2257	*/
2258	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2259	{
2260	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2261	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2262	PGM_LOCK_VOID(pVM);
2263
2264	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2265	PPGMPAGE pPage;
2266	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2267	if (RT_SUCCESS(rc))
2268	{
2269	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2270	if (RT_SUCCESS(rc))
2271	{
2272	# ifdef IN_RING3
2273	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2274	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2275	# else
2276	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2277	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2278	# endif
2279	PGM_UNLOCK(pVM);
2280	return VINF_SUCCESS;
2281	}
2282	AssertRC(rc);
2283	}
2284
2285	PGM_UNLOCK(pVM);
2286	*ppPdpt = NULL;
2287	return rc;
2288	}
2289
2290
2291	/**
2292	* Performs the lazy mapping / updating of a PAE guest PD.
2293	*
2294	* @returns Pointer to the mapping.
2295	* @returns VBox status code.
2296	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2297	* @param iPdpt Which PD entry to map (0..3).
2298	* @param ppPd Where to return the pointer to the mapping. This is
2299	* always set.
2300	*/
2301	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2302	{
2303	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2304	PGM_LOCK_VOID(pVM);
2305
2306	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2307	Assert(pGuestPDPT);
2308	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2309	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2310	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2311
2312	PPGMPAGE pPage;
2313	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2314	if (RT_SUCCESS(rc))
2315	{
2316	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2317	AssertRC(rc);
2318	if (RT_SUCCESS(rc))
2319	{
2320	# ifdef IN_RING3
2321	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2322	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2323	# else
2324	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2325	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2326	# endif
2327	if (fChanged)
2328	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2329	PGM_UNLOCK(pVM);
2330	return VINF_SUCCESS;
2331	}
2332	}
2333
2334	/* Invalid page or some failure, invalidate the entry. */
2335	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2336	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2337	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2338
2339	PGM_UNLOCK(pVM);
2340	return rc;
2341	}
2342
2343
2344	/**
2345	* Performs the lazy mapping of the 32-bit guest PD.
2346	*
2347	* @returns VBox status code.
2348	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2349	* @param ppPml4 Where to return the pointer to the mapping. This will
2350	* always be set.
2351	*/
2352	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2353	{
2354	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2355	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2356	PGM_LOCK_VOID(pVM);
2357
2358	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2359	PPGMPAGE pPage;
2360	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2361	if (RT_SUCCESS(rc))
2362	{
2363	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2364	if (RT_SUCCESS(rc))
2365	{
2366	# ifdef IN_RING3
2367	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2368	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2369	# else
2370	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2371	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2372	# endif
2373	PGM_UNLOCK(pVM);
2374	return VINF_SUCCESS;
2375	}
2376	}
2377
2378	PGM_UNLOCK(pVM);
2379	*ppPml4 = NULL;
2380	return rc;
2381	}
2382
2383
2384	/**
2385	* Gets the current CR3 register value for the shadow memory context.
2386	* @returns CR3 value.
2387	* @param pVCpu The cross context virtual CPU structure.
2388	*/
2389	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2390	{
2391	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2392	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2393	return pPoolPage->Core.Key;
2394	}
2395
2396
2397	/**
2398	* Forces lazy remapping of the guest's PAE page-directory structures.
2399	*
2400	* @param pVCpu The cross context virtual CPU structure.
2401	*/
2402	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2403	{
2404	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2405	{
2406	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2407	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2408	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2409	}
2410	}
2411
2412
2413	/**
2414	* Gets the PGM CR3 value masked according to the current guest mode.
2415	*
2416	* @returns The masked PGM CR3 value.
2417	* @param pVCpu The cross context virtual CPU structure.
2418	* @param uCr3 The raw guest CR3 value.
2419	*/
2420	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
2421	{
2422	RTGCPHYS GCPhysCR3;
2423	switch (pVCpu->pgm.s.enmGuestMode)
2424	{
2425	case PGMMODE_PAE:
2426	case PGMMODE_PAE_NX:
2427	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_PAE_PAGE_MASK);
2428	break;
2429	case PGMMODE_AMD64:
2430	case PGMMODE_AMD64_NX:
2431	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_AMD64_PAGE_MASK);
2432	break;
2433	default:
2434	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_PAGE_MASK);
2435	break;
2436	}
2437	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2438	return GCPhysCR3;
2439	}
2440
2441
2442	/**
2443	* Performs and schedules necessary updates following a CR3 load or reload.
2444	*
2445	* This will normally involve mapping the guest PD or nPDPT
2446	*
2447	* @returns VBox status code.
2448	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2449	* safely be ignored and overridden since the FF will be set too then.
2450	* @param pVCpu The cross context virtual CPU structure.
2451	* @param cr3 The new cr3.
2452	* @param fGlobal Indicates whether this is a global flush or not.
2453	* @param fPdpesMapped Whether the PAE PDPEs (and PDPT) have been mapped.
2454	*/
2455	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal, bool fPdpesMapped)
2456	{
2457	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2458	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2459
2460	VMCPU_ASSERT_EMT(pVCpu);
2461
2462	/*
2463	* Always flag the necessary updates; necessary for hardware acceleration
2464	*/
2465	/** @todo optimize this, it shouldn't always be necessary. */
2466	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2467	if (fGlobal)
2468	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2469	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
2470
2471	/*
2472	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2473	*/
2474	int rc = VINF_SUCCESS;
2475	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2476	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2477	if (GCPhysOldCR3 != GCPhysCR3)
2478	{
2479	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2480	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2481	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2482
2483	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2484	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, fPdpesMapped);
2485	if (RT_LIKELY(rc == VINF_SUCCESS))
2486	{
2487	if (pgmMapAreMappingsFloating(pVM))
2488	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2489	}
2490	else
2491	{
2492	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2493	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2494	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2495	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2496	if (pgmMapAreMappingsFloating(pVM))
2497	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MONITOR_CR3;
2498	}
2499
2500	if (fGlobal)
2501	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2502	else
2503	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2504	}
2505	else
2506	{
2507	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2508	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2509	if (pPool->cDirtyPages)
2510	{
2511	PGM_LOCK_VOID(pVM);
2512	pgmPoolResetDirtyPages(pVM);
2513	PGM_UNLOCK(pVM);
2514	}
2515	#endif
2516	/*
2517	* Check if we have a pending update of the CR3 monitoring.
2518	*/
2519	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2520	{
2521	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2522	Assert(!pVM->pgm.s.fMappingsFixed); Assert(pgmMapAreMappingsEnabled(pVM));
2523	}
2524	if (fGlobal)
2525	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2526	else
2527	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2528
2529	/*
2530	* Flush PAE PDPTEs.
2531	*/
2532	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2533	pgmGstFlushPaePdpes(pVCpu);
2534	}
2535
2536	IEMTlbInvalidateAll(pVCpu, false /fVmm/);
2537	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2538	return rc;
2539	}
2540
2541
2542	/**
2543	* Performs and schedules necessary updates following a CR3 load or reload when
2544	* using nested or extended paging.
2545	*
2546	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2547	* TLB and triggering a SyncCR3.
2548	*
2549	* This will normally involve mapping the guest PD or nPDPT
2550	*
2551	* @returns VBox status code.
2552	* @retval VINF_SUCCESS.
2553	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2554	* paging modes). This can safely be ignored and overridden since the
2555	* FF will be set too then.
2556	* @param pVCpu The cross context virtual CPU structure.
2557	* @param cr3 The new CR3.
2558	* @param fPdpesMapped Whether the PAE PDPEs (and PDPT) have been mapped.
2559	*/
2560	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3, bool fPdpesMapped)
2561	{
2562	VMCPU_ASSERT_EMT(pVCpu);
2563	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
2564
2565	/* We assume we're only called in nested paging mode. */
2566	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2567	Assert(!pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2568	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3));
2569
2570	/*
2571	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2572	*/
2573	int rc = VINF_SUCCESS;
2574	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2575	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2576	{
2577	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2578	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2579	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2580
2581	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2582	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, fPdpesMapped);
2583
2584	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2585	}
2586	/*
2587	* Flush PAE PDPTEs.
2588	*/
2589	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2590	pgmGstFlushPaePdpes(pVCpu);
2591
2592	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2593	return rc;
2594	}
2595
2596
2597	/**
2598	* Synchronize the paging structures.
2599	*
2600	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2601	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2602	* in several places, most importantly whenever the CR3 is loaded.
2603	*
2604	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2605	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2606	* the VMM into guest context.
2607	* @param pVCpu The cross context virtual CPU structure.
2608	* @param cr0 Guest context CR0 register
2609	* @param cr3 Guest context CR3 register
2610	* @param cr4 Guest context CR4 register
2611	* @param fGlobal Including global page directories or not
2612	*/
2613	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2614	{
2615	int rc;
2616
2617	VMCPU_ASSERT_EMT(pVCpu);
2618
2619	/*
2620	* The pool may have pending stuff and even require a return to ring-3 to
2621	* clear the whole thing.
2622	*/
2623	rc = pgmPoolSyncCR3(pVCpu);
2624	if (rc != VINF_SUCCESS)
2625	return rc;
2626
2627	/*
2628	* We might be called when we shouldn't.
2629	*
2630	* The mode switching will ensure that the PD is resynced after every mode
2631	* switch. So, if we find ourselves here when in protected or real mode
2632	* we can safely clear the FF and return immediately.
2633	*/
2634	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2635	{
2636	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2637	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2638	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2639	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2640	return VINF_SUCCESS;
2641	}
2642
2643	/* If global pages are not supported, then all flushes are global. */
2644	if (!(cr4 & X86_CR4_PGE))
2645	fGlobal = true;
2646	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2647	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2648
2649	/*
2650	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2651	* This should be done before SyncCR3.
2652	*/
2653	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2654	{
2655	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2656
2657	RTGCPHYS const GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
2658	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2659	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2660	{
2661	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2662	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2663	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2664	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2665	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, false /* fPdpesMapped */);
2666	}
2667
2668	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2669	if ( rc == VINF_PGM_SYNC_CR3
2670	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2671	{
2672	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2673	#ifdef IN_RING3
2674	rc = pgmPoolSyncCR3(pVCpu);
2675	#else
2676	if (rc == VINF_PGM_SYNC_CR3)
2677	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
2678	return VINF_PGM_SYNC_CR3;
2679	#endif
2680	}
2681	AssertRCReturn(rc, rc);
2682	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2683	}
2684
2685	/*
2686	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2687	*/
2688	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2689
2690	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2691	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2692	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2693	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2694
2695	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2696	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2697	if (rc == VINF_SUCCESS)
2698	{
2699	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2700	{
2701	/* Go back to ring 3 if a pgm pool sync is again pending. */
2702	return VINF_PGM_SYNC_CR3;
2703	}
2704
2705	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2706	{
2707	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2708	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2709	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2710	}
2711
2712	/*
2713	* Check if we have a pending update of the CR3 monitoring.
2714	*/
2715	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MONITOR_CR3)
2716	{
2717	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MONITOR_CR3;
2718	Assert(!pVCpu->CTX_SUFF(pVM)->pgm.s.fMappingsFixed);
2719	Assert(pgmMapAreMappingsEnabled(pVCpu->CTX_SUFF(pVM)));
2720	}
2721	}
2722
2723	/*
2724	* Now flush the CR3 (guest context).
2725	*/
2726	if (rc == VINF_SUCCESS)
2727	PGM_INVL_VCPU_TLBS(pVCpu);
2728	return rc;
2729	}
2730
2731
2732	/**
2733	* Maps all the PAE PDPE entries.
2734	*
2735	* @returns VBox status code.
2736	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2737	* @param paPaePdpes The new PAE PDPE values.
2738	*
2739	* @remarks This function may be invoked during the process of changing the guest
2740	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2741	* reflect PAE paging just yet.
2742	*/
2743	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2744	{
2745	Assert(paPaePdpes);
2746	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2747	{
2748	X86PDPE const PaePdpe = paPaePdpes[i];
2749
2750	/*
2751	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2752	* are deferred.[1] Also, different situations require different handling of invalid
2753	* PDPE entries. Here we assume the caller has already validated or doesn't require
2754	* validation of the PDPEs.
2755	*
2756	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2757	*/
2758	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2759	{
2760	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2761	RTHCPTR HCPtr;
2762	RTGCPHYS const GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2763
2764	PGM_LOCK_VOID(pVM);
2765	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2766	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2767	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2768	PGM_UNLOCK(pVM);
2769	if (RT_SUCCESS(rc))
2770	{
2771	# ifdef IN_RING3
2772	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2773	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2774	# else
2775	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2776	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2777	# endif
2778	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
2779	continue;
2780	}
2781	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
2782	}
2783	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2784	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2785	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2786	}
2787
2788	return VINF_SUCCESS;
2789	}
2790
2791
2792	/**
2793	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
2794	*
2795	* @returns VBox status code.
2796	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2797	* @param cr3 The guest CR3 value.
2798	*
2799	* @remarks This function may be invoked during the process of changing the guest
2800	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
2801	* PAE paging just yet.
2802	*/
2803	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
2804	{
2805	/*
2806	* Read the page-directory-pointer table (PDPT) at CR3.
2807	*/
2808	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2809	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
2810	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2811
2812	PGM_LOCK_VOID(pVM);
2813	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
2814	AssertReturnStmt(pPageCR3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
2815
2816	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
2817	RTHCPTR HCPtrGuestCr3;
2818	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3, (void **)&HCPtrGuestCr3);
2819	PGM_UNLOCK(pVM);
2820	AssertRCReturn(rc, rc);
2821	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
2822
2823	/*
2824	* Validate the page-directory-pointer table entries (PDPE).
2825	*/
2826	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
2827	{
2828	/*
2829	* Map the PDPT.
2830	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
2831	* that PGMFlushTLB will be called soon and only a change to CR3 then
2832	* will cause the shadow page tables to be updated.
2833	*/
2834	# ifdef IN_RING3
2835	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
2836	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2837	# else
2838	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2839	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
2840	# endif
2841
2842	/*
2843	* Update CPUM.
2844	* We do this prior to mapping the PDPEs to keep the order consistent
2845	* with what's used in HM. In practice, it doesn't really matter.
2846	*/
2847	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
2848
2849	/*
2850	* Map the PDPEs.
2851	*/
2852	return PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
2853	}
2854	return VERR_PGM_PAE_PDPE_RSVD;
2855	}
2856
2857
2858	/**
2859	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2860	*
2861	* @returns VBox status code, with the following informational code for
2862	* VM scheduling.
2863	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2864	* @retval VINF_PGM_CHANGE_MODE if we're in RC the mode changes. This will
2865	* NOT be returned in ring-3 or ring-0.
2866	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2867	*
2868	* @param pVCpu The cross context virtual CPU structure.
2869	* @param cr0 The new cr0.
2870	* @param cr4 The new cr4.
2871	* @param efer The new extended feature enable register.
2872	*/
2873	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
2874	{
2875	VMCPU_ASSERT_EMT(pVCpu);
2876
2877	/*
2878	* Calc the new guest mode.
2879	*
2880	* Note! We check PG before PE and without requiring PE because of the
2881	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2882	*/
2883	PGMMODE enmGuestMode;
2884	if (cr0 & X86_CR0_PG)
2885	{
2886	if (!(cr4 & X86_CR4_PAE))
2887	{
2888	bool const fPse = !!(cr4 & X86_CR4_PSE);
2889	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2890	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2891	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2892	enmGuestMode = PGMMODE_32_BIT;
2893	}
2894	else if (!(efer & MSR_K6_EFER_LME))
2895	{
2896	if (!(efer & MSR_K6_EFER_NXE))
2897	enmGuestMode = PGMMODE_PAE;
2898	else
2899	enmGuestMode = PGMMODE_PAE_NX;
2900	}
2901	else
2902	{
2903	if (!(efer & MSR_K6_EFER_NXE))
2904	enmGuestMode = PGMMODE_AMD64;
2905	else
2906	enmGuestMode = PGMMODE_AMD64_NX;
2907	}
2908	}
2909	else if (!(cr0 & X86_CR0_PE))
2910	enmGuestMode = PGMMODE_REAL;
2911	else
2912	enmGuestMode = PGMMODE_PROTECTED;
2913
2914	/*
2915	* Did it change?
2916	*/
2917	if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2918	return VINF_SUCCESS;
2919
2920	/* Flush the TLB */
2921	PGM_INVL_VCPU_TLBS(pVCpu);
2922	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
2923	}
2924
2925
2926	/**
2927	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
2928	*
2929	* @returns PGM_TYPE_*.
2930	* @param pgmMode The mode value to convert.
2931	*/
2932	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
2933	{
2934	switch (pgmMode)
2935	{
2936	case PGMMODE_REAL: return PGM_TYPE_REAL;
2937	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
2938	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
2939	case PGMMODE_PAE:
2940	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
2941	case PGMMODE_AMD64:
2942	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
2943	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
2944	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
2945	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
2946	case PGMMODE_EPT: return PGM_TYPE_EPT;
2947	case PGMMODE_NONE: return PGM_TYPE_NONE;
2948	default:
2949	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
2950	}
2951	}
2952
2953
2954	/**
2955	* Calculates the shadow paging mode.
2956	*
2957	* @returns The shadow paging mode.
2958	* @param pVM The cross context VM structure.
2959	* @param enmGuestMode The guest mode.
2960	* @param enmHostMode The host mode.
2961	* @param enmShadowMode The current shadow mode.
2962	*/
2963	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
2964	{
2965	switch (enmGuestMode)
2966	{
2967	/*
2968	* When switching to real or protected mode we don't change
2969	* anything since it's likely that we'll switch back pretty soon.
2970	*
2971	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
2972	* and is supposed to determine which shadow paging and switcher to
2973	* use during init.
2974	*/
2975	case PGMMODE_REAL:
2976	case PGMMODE_PROTECTED:
2977	if ( enmShadowMode != PGMMODE_INVALID
2978	&& VM_IS_RAW_MODE_ENABLED(pVM) /* always switch in hm and nem modes! */)
2979	break; /* (no change) */
2980
2981	switch (enmHostMode)
2982	{
2983	case SUPPAGINGMODE_32_BIT:
2984	case SUPPAGINGMODE_32_BIT_GLOBAL:
2985	enmShadowMode = PGMMODE_32_BIT;
2986	break;
2987
2988	case SUPPAGINGMODE_PAE:
2989	case SUPPAGINGMODE_PAE_NX:
2990	case SUPPAGINGMODE_PAE_GLOBAL:
2991	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2992	enmShadowMode = PGMMODE_PAE;
2993	break;
2994
2995	case SUPPAGINGMODE_AMD64:
2996	case SUPPAGINGMODE_AMD64_GLOBAL:
2997	case SUPPAGINGMODE_AMD64_NX:
2998	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2999	enmShadowMode = PGMMODE_PAE;
3000	break;
3001
3002	default:
3003	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3004	}
3005	break;
3006
3007	case PGMMODE_32_BIT:
3008	switch (enmHostMode)
3009	{
3010	case SUPPAGINGMODE_32_BIT:
3011	case SUPPAGINGMODE_32_BIT_GLOBAL:
3012	enmShadowMode = PGMMODE_32_BIT;
3013	break;
3014
3015	case SUPPAGINGMODE_PAE:
3016	case SUPPAGINGMODE_PAE_NX:
3017	case SUPPAGINGMODE_PAE_GLOBAL:
3018	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3019	enmShadowMode = PGMMODE_PAE;
3020	break;
3021
3022	case SUPPAGINGMODE_AMD64:
3023	case SUPPAGINGMODE_AMD64_GLOBAL:
3024	case SUPPAGINGMODE_AMD64_NX:
3025	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3026	enmShadowMode = PGMMODE_PAE;
3027	break;
3028
3029	default:
3030	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3031	}
3032	break;
3033
3034	case PGMMODE_PAE:
3035	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
3036	switch (enmHostMode)
3037	{
3038	case SUPPAGINGMODE_32_BIT:
3039	case SUPPAGINGMODE_32_BIT_GLOBAL:
3040	enmShadowMode = PGMMODE_PAE;
3041	break;
3042
3043	case SUPPAGINGMODE_PAE:
3044	case SUPPAGINGMODE_PAE_NX:
3045	case SUPPAGINGMODE_PAE_GLOBAL:
3046	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3047	enmShadowMode = PGMMODE_PAE;
3048	break;
3049
3050	case SUPPAGINGMODE_AMD64:
3051	case SUPPAGINGMODE_AMD64_GLOBAL:
3052	case SUPPAGINGMODE_AMD64_NX:
3053	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3054	enmShadowMode = PGMMODE_PAE;
3055	break;
3056
3057	default:
3058	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3059	}
3060	break;
3061
3062	case PGMMODE_AMD64:
3063	case PGMMODE_AMD64_NX:
3064	switch (enmHostMode)
3065	{
3066	case SUPPAGINGMODE_32_BIT:
3067	case SUPPAGINGMODE_32_BIT_GLOBAL:
3068	enmShadowMode = PGMMODE_AMD64;
3069	break;
3070
3071	case SUPPAGINGMODE_PAE:
3072	case SUPPAGINGMODE_PAE_NX:
3073	case SUPPAGINGMODE_PAE_GLOBAL:
3074	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3075	enmShadowMode = PGMMODE_AMD64;
3076	break;
3077
3078	case SUPPAGINGMODE_AMD64:
3079	case SUPPAGINGMODE_AMD64_GLOBAL:
3080	case SUPPAGINGMODE_AMD64_NX:
3081	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3082	enmShadowMode = PGMMODE_AMD64;
3083	break;
3084
3085	default:
3086	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3087	}
3088	break;
3089
3090	default:
3091	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3092	}
3093
3094	/*
3095	* Override the shadow mode when NEM or nested paging is active.
3096	*/
3097	if (VM_IS_NEM_ENABLED(pVM))
3098	{
3099	pVM->pgm.s.fNestedPaging = true;
3100	enmShadowMode = PGMMODE_NONE;
3101	}
3102	else
3103	{
3104	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3105	pVM->pgm.s.fNestedPaging = fNestedPaging;
3106	if (fNestedPaging)
3107	{
3108	if (HMIsVmxActive(pVM))
3109	enmShadowMode = PGMMODE_EPT;
3110	else
3111	{
3112	/* The nested SVM paging depends on the host one. */
3113	Assert(HMIsSvmActive(pVM));
3114	if ( enmGuestMode == PGMMODE_AMD64
3115	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3116	enmShadowMode = PGMMODE_NESTED_AMD64;
3117	else
3118	switch (pVM->pgm.s.enmHostMode)
3119	{
3120	case SUPPAGINGMODE_32_BIT:
3121	case SUPPAGINGMODE_32_BIT_GLOBAL:
3122	enmShadowMode = PGMMODE_NESTED_32BIT;
3123	break;
3124
3125	case SUPPAGINGMODE_PAE:
3126	case SUPPAGINGMODE_PAE_GLOBAL:
3127	case SUPPAGINGMODE_PAE_NX:
3128	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3129	enmShadowMode = PGMMODE_NESTED_PAE;
3130	break;
3131
3132	case SUPPAGINGMODE_AMD64:
3133	case SUPPAGINGMODE_AMD64_GLOBAL:
3134	case SUPPAGINGMODE_AMD64_NX:
3135	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3136	enmShadowMode = PGMMODE_NESTED_AMD64;
3137	break;
3138
3139	default:
3140	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3141	}
3142	}
3143	}
3144	}
3145
3146	return enmShadowMode;
3147	}
3148
3149
3150	/**
3151	* Performs the actual mode change.
3152	* This is called by PGMChangeMode and pgmR3InitPaging().
3153	*
3154	* @returns VBox status code. May suspend or power off the VM on error, but this
3155	* will trigger using FFs and not informational status codes.
3156	*
3157	* @param pVM The cross context VM structure.
3158	* @param pVCpu The cross context virtual CPU structure.
3159	* @param enmGuestMode The new guest mode. This is assumed to be different from
3160	* the current mode.
3161	*/
3162	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode)
3163	{
3164	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3165	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3166
3167	/*
3168	* Calc the shadow mode and switcher.
3169	*/
3170	PGMMODE enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3171
3172	/*
3173	* Exit old mode(s).
3174	*/
3175	/* shadow */
3176	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3177	{
3178	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3179	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3180	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3181	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3182	{
3183	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3184	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3185	}
3186	}
3187	else
3188	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3189
3190	/* guest */
3191	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3192	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3193	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3194	{
3195	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3196	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3197	}
3198	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3199
3200	/*
3201	* Change the paging mode data indexes.
3202	*/
3203	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3204	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3205	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3206	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3207	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3208	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3209	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3210	#ifdef IN_RING3
3211	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3212	#endif
3213
3214	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3215	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3216	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3217	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3218	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3219	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3220	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3221	#ifdef IN_RING3
3222	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3223	#endif
3224
3225	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3226	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3227	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3228	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3229	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3230	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3231	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3232	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3233	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3234	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3235	#ifdef VBOX_STRICT
3236	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3237	#endif
3238
3239	/*
3240	* Enter new shadow mode (if changed).
3241	*/
3242	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3243	{
3244	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3245	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
3246	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3247	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3248	}
3249
3250	/*
3251	* Always flag the necessary updates
3252	*/
3253	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3254
3255	/*
3256	* Enter the new guest and shadow+guest modes.
3257	*/
3258	/* Calc the new CR3 value. */
3259	RTGCPHYS GCPhysCR3;
3260	switch (enmGuestMode)
3261	{
3262	case PGMMODE_REAL:
3263	case PGMMODE_PROTECTED:
3264	GCPhysCR3 = NIL_RTGCPHYS;
3265	break;
3266
3267	case PGMMODE_32_BIT:
3268	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3269	break;
3270
3271	case PGMMODE_PAE_NX:
3272	case PGMMODE_PAE:
3273	if (!pVM->cpum.ro.GuestFeatures.fPae)
3274	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3275	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3276	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3277	#else
3278	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3279
3280	#endif
3281	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3282	break;
3283
3284	#ifdef VBOX_WITH_64_BITS_GUESTS
3285	case PGMMODE_AMD64_NX:
3286	case PGMMODE_AMD64:
3287	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3288	break;
3289	#endif
3290	default:
3291	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3292	}
3293
3294	/* Enter the new guest mode. */
3295	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3296	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3297	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3298
3299	/* Set the new guest CR3. */
3300	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3301
3302	/* status codes. */
3303	AssertRC(rc);
3304	AssertRC(rc2);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = rc2;
3308	if (RT_SUCCESS(rc)) /* no informational status codes. */
3309	rc = VINF_SUCCESS;
3310	}
3311
3312	/*
3313	* Notify HM.
3314	*/
3315	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3316	return rc;
3317	}
3318
3319
3320	/**
3321	* Called by CPUM or REM when CR0.WP changes to 1.
3322	*
3323	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3324	* @thread EMT
3325	*/
3326	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3327	{
3328	/*
3329	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3330	*
3331	* Use the counter to judge whether there might be pool pages with active
3332	* hacks in them. If there are, we will be running the risk of messing up
3333	* the guest by allowing it to write to read-only pages. Thus, we have to
3334	* clear the page pool ASAP if there is the slightest chance.
3335	*/
3336	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3337	{
3338	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3339
3340	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3341	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3342	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3343	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3344	}
3345	}
3346
3347
3348	/**
3349	* Gets the current guest paging mode.
3350	*
3351	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3352	*
3353	* @returns The current paging mode.
3354	* @param pVCpu The cross context virtual CPU structure.
3355	*/
3356	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3357	{
3358	return pVCpu->pgm.s.enmGuestMode;
3359	}
3360
3361
3362	/**
3363	* Gets the current shadow paging mode.
3364	*
3365	* @returns The current paging mode.
3366	* @param pVCpu The cross context virtual CPU structure.
3367	*/
3368	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3369	{
3370	return pVCpu->pgm.s.enmShadowMode;
3371	}
3372
3373
3374	/**
3375	* Gets the current host paging mode.
3376	*
3377	* @returns The current paging mode.
3378	* @param pVM The cross context VM structure.
3379	*/
3380	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3381	{
3382	switch (pVM->pgm.s.enmHostMode)
3383	{
3384	case SUPPAGINGMODE_32_BIT:
3385	case SUPPAGINGMODE_32_BIT_GLOBAL:
3386	return PGMMODE_32_BIT;
3387
3388	case SUPPAGINGMODE_PAE:
3389	case SUPPAGINGMODE_PAE_GLOBAL:
3390	return PGMMODE_PAE;
3391
3392	case SUPPAGINGMODE_PAE_NX:
3393	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3394	return PGMMODE_PAE_NX;
3395
3396	case SUPPAGINGMODE_AMD64:
3397	case SUPPAGINGMODE_AMD64_GLOBAL:
3398	return PGMMODE_AMD64;
3399
3400	case SUPPAGINGMODE_AMD64_NX:
3401	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3402	return PGMMODE_AMD64_NX;
3403
3404	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3405	}
3406
3407	return PGMMODE_INVALID;
3408	}
3409
3410
3411	/**
3412	* Get mode name.
3413	*
3414	* @returns read-only name string.
3415	* @param enmMode The mode which name is desired.
3416	*/
3417	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3418	{
3419	switch (enmMode)
3420	{
3421	case PGMMODE_REAL: return "Real";
3422	case PGMMODE_PROTECTED: return "Protected";
3423	case PGMMODE_32_BIT: return "32-bit";
3424	case PGMMODE_PAE: return "PAE";
3425	case PGMMODE_PAE_NX: return "PAE+NX";
3426	case PGMMODE_AMD64: return "AMD64";
3427	case PGMMODE_AMD64_NX: return "AMD64+NX";
3428	case PGMMODE_NESTED_32BIT: return "Nested-32";
3429	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3430	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3431	case PGMMODE_EPT: return "EPT";
3432	case PGMMODE_NONE: return "None";
3433	default: return "unknown mode value";
3434	}
3435	}
3436
3437
3438	/**
3439	* Gets the physical address represented in the guest CR3 as PGM sees it.
3440	*
3441	* This is mainly for logging and debugging.
3442	*
3443	* @returns PGM's guest CR3 value.
3444	* @param pVCpu The cross context virtual CPU structure.
3445	*/
3446	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3447	{
3448	return pVCpu->pgm.s.GCPhysCR3;
3449	}
3450
3451
3452
3453	/**
3454	* Notification from CPUM that the EFER.NXE bit has changed.
3455	*
3456	* @param pVCpu The cross context virtual CPU structure of the CPU for
3457	* which EFER changed.
3458	* @param fNxe The new NXE state.
3459	*/
3460	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3461	{
3462	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3463	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3464
3465	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3466	if (fNxe)
3467	{
3468	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3469	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3470	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3471	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3472	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3473	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3474	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3475	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3476	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3477	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3478	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3479
3480	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3481	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3482	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3483	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3484	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3485	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3486	}
3487	else
3488	{
3489	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3490	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3491	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3492	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3493	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3494	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3495	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3496	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3497	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3498	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3499	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3500
3501	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3502	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3503	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3504	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3505	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3506	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3507	}
3508	}
3509
3510
3511	/**
3512	* Check if any pgm pool pages are marked dirty (not monitored)
3513	*
3514	* @returns bool locked/not locked
3515	* @param pVM The cross context VM structure.
3516	*/
3517	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3518	{
3519	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3520	}
3521
3522
3523	/**
3524	* Check if this VCPU currently owns the PGM lock.
3525	*
3526	* @returns bool owner/not owner
3527	* @param pVM The cross context VM structure.
3528	*/
3529	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3530	{
3531	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3532	}
3533
3534
3535	/**
3536	* Enable or disable large page usage
3537	*
3538	* @returns VBox status code.
3539	* @param pVM The cross context VM structure.
3540	* @param fUseLargePages Use/not use large pages
3541	*/
3542	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3543	{
3544	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3545
3546	pVM->pgm.s.fUseLargePages = fUseLargePages;
3547	return VINF_SUCCESS;
3548	}
3549
3550
3551	/**
3552	* Acquire the PGM lock.
3553	*
3554	* @returns VBox status code
3555	* @param pVM The cross context VM structure.
3556	* @param fVoid Set if the caller cannot handle failure returns.
3557	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3558	*/
3559	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3560	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3561	#else
3562	int pgmLock(PVMCC pVM, bool fVoid)
3563	#endif
3564	{
3565	#if defined(VBOX_STRICT)
3566	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3567	#else
3568	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3569	#endif
3570	if (RT_SUCCESS(rc))
3571	return rc;
3572	if (fVoid)
3573	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3574	else
3575	AssertRC(rc);
3576	return rc;
3577	}
3578
3579
3580	/**
3581	* Release the PGM lock.
3582	*
3583	* @returns VBox status code
3584	* @param pVM The cross context VM structure.
3585	*/
3586	void pgmUnlock(PVMCC pVM)
3587	{
3588	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3589	pVM->pgm.s.cDeprecatedPageLocks = 0;
3590	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3591	if (rc == VINF_SEM_NESTED)
3592	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3593	}
3594
3595
3596	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3597
3598	/** Format handler for PGMPAGE.
3599	* @copydoc FNRTSTRFORMATTYPE */
3600	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3601	const char pszType, void const pvValue,
3602	int cchWidth, int cchPrecision, unsigned fFlags,
3603	void *pvUser)
3604	{
3605	size_t cch;
3606	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3607	if (RT_VALID_PTR(pPage))
3608	{
3609	char szTmp[64+80];
3610
3611	cch = 0;
3612
3613	/* The single char state stuff. */
3614	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3615	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3616
3617	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3618	if (IS_PART_INCLUDED(5))
3619	{
3620	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3621	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3622	}
3623
3624	/* The type. */
3625	if (IS_PART_INCLUDED(4))
3626	{
3627	szTmp[cch++] = ':';
3628	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3629	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3630	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3631	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3632	}
3633
3634	/* The numbers. */
3635	if (IS_PART_INCLUDED(3))
3636	{
3637	szTmp[cch++] = ':';
3638	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3639	}
3640
3641	if (IS_PART_INCLUDED(2))
3642	{
3643	szTmp[cch++] = ':';
3644	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3645	}
3646
3647	if (IS_PART_INCLUDED(6))
3648	{
3649	szTmp[cch++] = ':';
3650	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3651	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3652	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3653	}
3654	# undef IS_PART_INCLUDED
3655
3656	cch = pfnOutput(pvArgOutput, szTmp, cch);
3657	}
3658	else
3659	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3660	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3661	return cch;
3662	}
3663
3664
3665	/** Format handler for PGMRAMRANGE.
3666	* @copydoc FNRTSTRFORMATTYPE */
3667	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3668	const char pszType, void const pvValue,
3669	int cchWidth, int cchPrecision, unsigned fFlags,
3670	void *pvUser)
3671	{
3672	size_t cch;
3673	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3674	if (RT_VALID_PTR(pRam))
3675	{
3676	char szTmp[80];
3677	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3678	cch = pfnOutput(pvArgOutput, szTmp, cch);
3679	}
3680	else
3681	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3682	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3683	return cch;
3684	}
3685
3686	/** Format type andlers to be registered/deregistered. */
3687	static const struct
3688	{
3689	char szType[24];
3690	PFNRTSTRFORMATTYPE pfnHandler;
3691	} g_aPgmFormatTypes[] =
3692	{
3693	{ "pgmpage", pgmFormatTypeHandlerPage },
3694	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3695	};
3696
3697	#endif /* !IN_R0 \|\| LOG_ENABLED */
3698
3699	/**
3700	* Registers the global string format types.
3701	*
3702	* This should be called at module load time or in some other manner that ensure
3703	* that it's called exactly one time.
3704	*
3705	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3706	*/
3707	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3708	{
3709	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3710	int rc = VINF_SUCCESS;
3711	unsigned i;
3712	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3713	{
3714	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3715	# ifdef IN_RING0
3716	if (rc == VERR_ALREADY_EXISTS)
3717	{
3718	/* in case of cleanup failure in ring-0 */
3719	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3720	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3721	}
3722	# endif
3723	}
3724	if (RT_FAILURE(rc))
3725	while (i-- > 0)
3726	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3727
3728	return rc;
3729	#else
3730	return VINF_SUCCESS;
3731	#endif
3732	}
3733
3734
3735	/**
3736	* Deregisters the global string format types.
3737	*
3738	* This should be called at module unload time or in some other manner that
3739	* ensure that it's called exactly one time.
3740	*/
3741	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3742	{
3743	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3744	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3745	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3746	#endif
3747	}
3748
3749	#ifdef VBOX_STRICT
3750
3751	# ifndef PGM_WITHOUT_MAPPINGS
3752	/**
3753	* Asserts that there are no mapping conflicts.
3754	*
3755	* @returns Number of conflicts.
3756	* @param pVM The cross context VM structure.
3757	*/
3758	VMMDECL(unsigned) PGMAssertNoMappingConflicts(PVM pVM)
3759	{
3760	unsigned cErrors = 0;
3761
3762	/* Only applies to raw mode -> 1 VPCU */
3763	Assert(pVM->cCpus == 1);
3764	PVMCPU pVCpu = &VMCC_GET_CPU_0(pVM);
3765
3766	/*
3767	* Check for mapping conflicts.
3768	*/
3769	for (PPGMMAPPING pMapping = pVM->pgm.s.CTX_SUFF(pMappings);
3770	pMapping;
3771	pMapping = pMapping->CTX_SUFF(pNext))
3772	{
3773	/** @todo This is slow and should be optimized, but since it's just assertions I don't care now. */
3774	for (RTGCPTR GCPtr = pMapping->GCPtr; GCPtr <= pMapping->GCPtrLast; GCPtr += PAGE_SIZE)
3775	{
3776	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)GCPtr, NULL, NULL);
3777	if (rc != VERR_PAGE_TABLE_NOT_PRESENT)
3778	{
3779	AssertMsgFailed(("Conflict at %RGv with %s\n", GCPtr, R3STRING(pMapping->pszDesc)));
3780	cErrors++;
3781	break;
3782	}
3783	}
3784	}
3785
3786	return cErrors;
3787	}
3788	# endif /* !PGM_WITHOUT_MAPPINGS */
3789
3790
3791	/**
3792	* Asserts that everything related to the guest CR3 is correctly shadowed.
3793	*
3794	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3795	* and assert the correctness of the guest CR3 mapping before asserting that the
3796	* shadow page tables is in sync with the guest page tables.
3797	*
3798	* @returns Number of conflicts.
3799	* @param pVM The cross context VM structure.
3800	* @param pVCpu The cross context virtual CPU structure.
3801	* @param cr3 The current guest CR3 register value.
3802	* @param cr4 The current guest CR4 register value.
3803	*/
3804	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3805	{
3806	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3807
3808	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3809	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3810	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3811
3812	PGM_LOCK_VOID(pVM);
3813	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3814	PGM_UNLOCK(pVM);
3815
3816	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3817	return cErrors;
3818	}
3819
3820	#endif /* VBOX_STRICT */
3821

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 91712

Download in other formats: