PGMAll.cpp@ 91854

Last change on this file since 91854 was 91854, checked in by vboxsync, 4 years ago
VMM: Removed PGM_WITHOUT_MAPPINGS and associated mapping code. bugref:9517
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 132.9 KB

Line
1	/* $Id: PGMAll.cpp 91854 2021-10-20 00:50:11Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor - All context code.
4	*/
5
6	/*
7	* Copyright (C) 2006-2020 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.215389.xyz. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_PGM
23	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
24	#include <VBox/vmm/pgm.h>
25	#include <VBox/vmm/cpum.h>
26	#include <VBox/vmm/selm.h>
27	#include <VBox/vmm/iem.h>
28	#include <VBox/vmm/iom.h>
29	#include <VBox/sup.h>
30	#include <VBox/vmm/mm.h>
31	#include <VBox/vmm/stam.h>
32	#include <VBox/vmm/trpm.h>
33	#include <VBox/vmm/em.h>
34	#include <VBox/vmm/hm.h>
35	#include <VBox/vmm/hm_vmx.h>
36	#include "PGMInternal.h"
37	#include <VBox/vmm/vmcc.h>
38	#include "PGMInline.h"
39	#include <iprt/assert.h>
40	#include <iprt/asm-amd64-x86.h>
41	#include <iprt/string.h>
42	#include <VBox/log.h>
43	#include <VBox/param.h>
44	#include <VBox/err.h>
45
46
47	/*********************************************************************************************************************************
48	* Internal Functions *
49	*********************************************************************************************************************************/
50	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD);
51	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde);
52	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD);
53	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD);
54
55
56	/*
57	* Shadow - 32-bit mode
58	*/
59	#define PGM_SHW_TYPE PGM_TYPE_32BIT
60	#define PGM_SHW_NAME(name) PGM_SHW_NAME_32BIT(name)
61	#include "PGMAllShw.h"
62
63	/* Guest - real mode */
64	#define PGM_GST_TYPE PGM_TYPE_REAL
65	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
66	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_REAL(name)
67	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
68	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
69	#include "PGMGstDefs.h"
70	#include "PGMAllGst.h"
71	#include "PGMAllBth.h"
72	#undef BTH_PGMPOOLKIND_PT_FOR_PT
73	#undef BTH_PGMPOOLKIND_ROOT
74	#undef PGM_BTH_NAME
75	#undef PGM_GST_TYPE
76	#undef PGM_GST_NAME
77
78	/* Guest - protected mode */
79	#define PGM_GST_TYPE PGM_TYPE_PROT
80	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
81	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_PROT(name)
82	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_PHYS
83	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD_PHYS
84	#include "PGMGstDefs.h"
85	#include "PGMAllGst.h"
86	#include "PGMAllBth.h"
87	#undef BTH_PGMPOOLKIND_PT_FOR_PT
88	#undef BTH_PGMPOOLKIND_ROOT
89	#undef PGM_BTH_NAME
90	#undef PGM_GST_TYPE
91	#undef PGM_GST_NAME
92
93	/* Guest - 32-bit mode */
94	#define PGM_GST_TYPE PGM_TYPE_32BIT
95	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
96	#define PGM_BTH_NAME(name) PGM_BTH_NAME_32BIT_32BIT(name)
97	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_32BIT_PT_FOR_32BIT_PT
98	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_32BIT_PT_FOR_32BIT_4MB
99	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_32BIT_PD
100	#include "PGMGstDefs.h"
101	#include "PGMAllGst.h"
102	#include "PGMAllBth.h"
103	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
104	#undef BTH_PGMPOOLKIND_PT_FOR_PT
105	#undef BTH_PGMPOOLKIND_ROOT
106	#undef PGM_BTH_NAME
107	#undef PGM_GST_TYPE
108	#undef PGM_GST_NAME
109
110	#undef PGM_SHW_TYPE
111	#undef PGM_SHW_NAME
112
113
114	/*
115	* Shadow - PAE mode
116	*/
117	#define PGM_SHW_TYPE PGM_TYPE_PAE
118	#define PGM_SHW_NAME(name) PGM_SHW_NAME_PAE(name)
119	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
120	#include "PGMAllShw.h"
121
122	/* Guest - real mode */
123	#define PGM_GST_TYPE PGM_TYPE_REAL
124	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
125	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_REAL(name)
126	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
127	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
128	#include "PGMGstDefs.h"
129	#include "PGMAllBth.h"
130	#undef BTH_PGMPOOLKIND_PT_FOR_PT
131	#undef BTH_PGMPOOLKIND_ROOT
132	#undef PGM_BTH_NAME
133	#undef PGM_GST_TYPE
134	#undef PGM_GST_NAME
135
136	/* Guest - protected mode */
137	#define PGM_GST_TYPE PGM_TYPE_PROT
138	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
139	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PROT(name)
140	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
141	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_PHYS
142	#include "PGMGstDefs.h"
143	#include "PGMAllBth.h"
144	#undef BTH_PGMPOOLKIND_PT_FOR_PT
145	#undef BTH_PGMPOOLKIND_ROOT
146	#undef PGM_BTH_NAME
147	#undef PGM_GST_TYPE
148	#undef PGM_GST_NAME
149
150	/* Guest - 32-bit mode */
151	#define PGM_GST_TYPE PGM_TYPE_32BIT
152	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
153	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_32BIT(name)
154	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_32BIT_PT
155	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_32BIT_4MB
156	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT_FOR_32BIT
157	#include "PGMGstDefs.h"
158	#include "PGMAllBth.h"
159	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
160	#undef BTH_PGMPOOLKIND_PT_FOR_PT
161	#undef BTH_PGMPOOLKIND_ROOT
162	#undef PGM_BTH_NAME
163	#undef PGM_GST_TYPE
164	#undef PGM_GST_NAME
165
166
167	/* Guest - PAE mode */
168	#define PGM_GST_TYPE PGM_TYPE_PAE
169	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
170	#define PGM_BTH_NAME(name) PGM_BTH_NAME_PAE_PAE(name)
171	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
172	#define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
173	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PDPT
174	#include "PGMGstDefs.h"
175	#include "PGMAllGst.h"
176	#include "PGMAllBth.h"
177	#undef BTH_PGMPOOLKIND_PT_FOR_BIG
178	#undef BTH_PGMPOOLKIND_PT_FOR_PT
179	#undef BTH_PGMPOOLKIND_ROOT
180	#undef PGM_BTH_NAME
181	#undef PGM_GST_TYPE
182	#undef PGM_GST_NAME
183
184	#undef PGM_SHW_TYPE
185	#undef PGM_SHW_NAME
186
187
188	/*
189	* Shadow - AMD64 mode
190	*/
191	#define PGM_SHW_TYPE PGM_TYPE_AMD64
192	#define PGM_SHW_NAME(name) PGM_SHW_NAME_AMD64(name)
193	#include "PGMAllShw.h"
194
195	/* Guest - protected mode (only used for AMD-V nested paging in 64 bits mode) */
196	/** @todo retire this hack. */
197	#define PGM_GST_TYPE PGM_TYPE_PROT
198	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
199	#define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_PROT(name)
200	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PHYS
201	#define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_PAE_PD_PHYS
202	#include "PGMGstDefs.h"
203	#include "PGMAllBth.h"
204	#undef BTH_PGMPOOLKIND_PT_FOR_PT
205	#undef BTH_PGMPOOLKIND_ROOT
206	#undef PGM_BTH_NAME
207	#undef PGM_GST_TYPE
208	#undef PGM_GST_NAME
209
210	#ifdef VBOX_WITH_64_BITS_GUESTS
211	/* Guest - AMD64 mode */
212	# define PGM_GST_TYPE PGM_TYPE_AMD64
213	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
214	# define PGM_BTH_NAME(name) PGM_BTH_NAME_AMD64_AMD64(name)
215	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_PAE_PT_FOR_PAE_PT
216	# define BTH_PGMPOOLKIND_PT_FOR_BIG PGMPOOLKIND_PAE_PT_FOR_PAE_2MB
217	# define BTH_PGMPOOLKIND_ROOT PGMPOOLKIND_64BIT_PML4
218	# include "PGMGstDefs.h"
219	# include "PGMAllGst.h"
220	# include "PGMAllBth.h"
221	# undef BTH_PGMPOOLKIND_PT_FOR_BIG
222	# undef BTH_PGMPOOLKIND_PT_FOR_PT
223	# undef BTH_PGMPOOLKIND_ROOT
224	# undef PGM_BTH_NAME
225	# undef PGM_GST_TYPE
226	# undef PGM_GST_NAME
227	#endif /* VBOX_WITH_64_BITS_GUESTS */
228
229	#undef PGM_SHW_TYPE
230	#undef PGM_SHW_NAME
231
232
233	/*
234	* Shadow - 32-bit nested paging mode.
235	*/
236	#define PGM_SHW_TYPE PGM_TYPE_NESTED_32BIT
237	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_32BIT(name)
238	#include "PGMAllShw.h"
239
240	/* Guest - real mode */
241	#define PGM_GST_TYPE PGM_TYPE_REAL
242	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
243	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_REAL(name)
244	#include "PGMGstDefs.h"
245	#include "PGMAllBth.h"
246	#undef PGM_BTH_NAME
247	#undef PGM_GST_TYPE
248	#undef PGM_GST_NAME
249
250	/* Guest - protected mode */
251	#define PGM_GST_TYPE PGM_TYPE_PROT
252	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
253	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PROT(name)
254	#include "PGMGstDefs.h"
255	#include "PGMAllBth.h"
256	#undef PGM_BTH_NAME
257	#undef PGM_GST_TYPE
258	#undef PGM_GST_NAME
259
260	/* Guest - 32-bit mode */
261	#define PGM_GST_TYPE PGM_TYPE_32BIT
262	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
263	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_32BIT(name)
264	#include "PGMGstDefs.h"
265	#include "PGMAllBth.h"
266	#undef PGM_BTH_NAME
267	#undef PGM_GST_TYPE
268	#undef PGM_GST_NAME
269
270	/* Guest - PAE mode */
271	#define PGM_GST_TYPE PGM_TYPE_PAE
272	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
273	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_PAE(name)
274	#include "PGMGstDefs.h"
275	#include "PGMAllBth.h"
276	#undef PGM_BTH_NAME
277	#undef PGM_GST_TYPE
278	#undef PGM_GST_NAME
279
280	#ifdef VBOX_WITH_64_BITS_GUESTS
281	/* Guest - AMD64 mode */
282	# define PGM_GST_TYPE PGM_TYPE_AMD64
283	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
284	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_32BIT_AMD64(name)
285	# include "PGMGstDefs.h"
286	# include "PGMAllBth.h"
287	# undef PGM_BTH_NAME
288	# undef PGM_GST_TYPE
289	# undef PGM_GST_NAME
290	#endif /* VBOX_WITH_64_BITS_GUESTS */
291
292	#undef PGM_SHW_TYPE
293	#undef PGM_SHW_NAME
294
295
296	/*
297	* Shadow - PAE nested paging mode.
298	*/
299	#define PGM_SHW_TYPE PGM_TYPE_NESTED_PAE
300	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_PAE(name)
301	#include "PGMAllShw.h"
302
303	/* Guest - real mode */
304	#define PGM_GST_TYPE PGM_TYPE_REAL
305	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
306	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_REAL(name)
307	#include "PGMGstDefs.h"
308	#include "PGMAllBth.h"
309	#undef PGM_BTH_NAME
310	#undef PGM_GST_TYPE
311	#undef PGM_GST_NAME
312
313	/* Guest - protected mode */
314	#define PGM_GST_TYPE PGM_TYPE_PROT
315	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
316	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PROT(name)
317	#include "PGMGstDefs.h"
318	#include "PGMAllBth.h"
319	#undef PGM_BTH_NAME
320	#undef PGM_GST_TYPE
321	#undef PGM_GST_NAME
322
323	/* Guest - 32-bit mode */
324	#define PGM_GST_TYPE PGM_TYPE_32BIT
325	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
326	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_32BIT(name)
327	#include "PGMGstDefs.h"
328	#include "PGMAllBth.h"
329	#undef PGM_BTH_NAME
330	#undef PGM_GST_TYPE
331	#undef PGM_GST_NAME
332
333	/* Guest - PAE mode */
334	#define PGM_GST_TYPE PGM_TYPE_PAE
335	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
336	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_PAE(name)
337	#include "PGMGstDefs.h"
338	#include "PGMAllBth.h"
339	#undef PGM_BTH_NAME
340	#undef PGM_GST_TYPE
341	#undef PGM_GST_NAME
342
343	#ifdef VBOX_WITH_64_BITS_GUESTS
344	/* Guest - AMD64 mode */
345	# define PGM_GST_TYPE PGM_TYPE_AMD64
346	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
347	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_PAE_AMD64(name)
348	# include "PGMGstDefs.h"
349	# include "PGMAllBth.h"
350	# undef PGM_BTH_NAME
351	# undef PGM_GST_TYPE
352	# undef PGM_GST_NAME
353	#endif /* VBOX_WITH_64_BITS_GUESTS */
354
355	#undef PGM_SHW_TYPE
356	#undef PGM_SHW_NAME
357
358
359	/*
360	* Shadow - AMD64 nested paging mode.
361	*/
362	#define PGM_SHW_TYPE PGM_TYPE_NESTED_AMD64
363	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NESTED_AMD64(name)
364	#include "PGMAllShw.h"
365
366	/* Guest - real mode */
367	#define PGM_GST_TYPE PGM_TYPE_REAL
368	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
369	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_REAL(name)
370	#include "PGMGstDefs.h"
371	#include "PGMAllBth.h"
372	#undef PGM_BTH_NAME
373	#undef PGM_GST_TYPE
374	#undef PGM_GST_NAME
375
376	/* Guest - protected mode */
377	#define PGM_GST_TYPE PGM_TYPE_PROT
378	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
379	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PROT(name)
380	#include "PGMGstDefs.h"
381	#include "PGMAllBth.h"
382	#undef PGM_BTH_NAME
383	#undef PGM_GST_TYPE
384	#undef PGM_GST_NAME
385
386	/* Guest - 32-bit mode */
387	#define PGM_GST_TYPE PGM_TYPE_32BIT
388	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
389	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_32BIT(name)
390	#include "PGMGstDefs.h"
391	#include "PGMAllBth.h"
392	#undef PGM_BTH_NAME
393	#undef PGM_GST_TYPE
394	#undef PGM_GST_NAME
395
396	/* Guest - PAE mode */
397	#define PGM_GST_TYPE PGM_TYPE_PAE
398	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
399	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_PAE(name)
400	#include "PGMGstDefs.h"
401	#include "PGMAllBth.h"
402	#undef PGM_BTH_NAME
403	#undef PGM_GST_TYPE
404	#undef PGM_GST_NAME
405
406	#ifdef VBOX_WITH_64_BITS_GUESTS
407	/* Guest - AMD64 mode */
408	# define PGM_GST_TYPE PGM_TYPE_AMD64
409	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
410	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NESTED_AMD64_AMD64(name)
411	# include "PGMGstDefs.h"
412	# include "PGMAllBth.h"
413	# undef PGM_BTH_NAME
414	# undef PGM_GST_TYPE
415	# undef PGM_GST_NAME
416	#endif /* VBOX_WITH_64_BITS_GUESTS */
417
418	#undef PGM_SHW_TYPE
419	#undef PGM_SHW_NAME
420
421
422	/*
423	* Shadow - EPT.
424	*/
425	#define PGM_SHW_TYPE PGM_TYPE_EPT
426	#define PGM_SHW_NAME(name) PGM_SHW_NAME_EPT(name)
427	#include "PGMAllShw.h"
428
429	/* Guest - real mode */
430	#define PGM_GST_TYPE PGM_TYPE_REAL
431	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
432	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_REAL(name)
433	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
434	#include "PGMGstDefs.h"
435	#include "PGMAllBth.h"
436	#undef BTH_PGMPOOLKIND_PT_FOR_PT
437	#undef PGM_BTH_NAME
438	#undef PGM_GST_TYPE
439	#undef PGM_GST_NAME
440
441	/* Guest - protected mode */
442	#define PGM_GST_TYPE PGM_TYPE_PROT
443	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
444	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PROT(name)
445	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
446	#include "PGMGstDefs.h"
447	#include "PGMAllBth.h"
448	#undef BTH_PGMPOOLKIND_PT_FOR_PT
449	#undef PGM_BTH_NAME
450	#undef PGM_GST_TYPE
451	#undef PGM_GST_NAME
452
453	/* Guest - 32-bit mode */
454	#define PGM_GST_TYPE PGM_TYPE_32BIT
455	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
456	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_32BIT(name)
457	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
458	#include "PGMGstDefs.h"
459	#include "PGMAllBth.h"
460	#undef BTH_PGMPOOLKIND_PT_FOR_PT
461	#undef PGM_BTH_NAME
462	#undef PGM_GST_TYPE
463	#undef PGM_GST_NAME
464
465	/* Guest - PAE mode */
466	#define PGM_GST_TYPE PGM_TYPE_PAE
467	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
468	#define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_PAE(name)
469	#define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
470	#include "PGMGstDefs.h"
471	#include "PGMAllBth.h"
472	#undef BTH_PGMPOOLKIND_PT_FOR_PT
473	#undef PGM_BTH_NAME
474	#undef PGM_GST_TYPE
475	#undef PGM_GST_NAME
476
477	#ifdef VBOX_WITH_64_BITS_GUESTS
478	/* Guest - AMD64 mode */
479	# define PGM_GST_TYPE PGM_TYPE_AMD64
480	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
481	# define PGM_BTH_NAME(name) PGM_BTH_NAME_EPT_AMD64(name)
482	# define BTH_PGMPOOLKIND_PT_FOR_PT PGMPOOLKIND_EPT_PT_FOR_PHYS
483	# include "PGMGstDefs.h"
484	# include "PGMAllBth.h"
485	# undef BTH_PGMPOOLKIND_PT_FOR_PT
486	# undef PGM_BTH_NAME
487	# undef PGM_GST_TYPE
488	# undef PGM_GST_NAME
489	#endif /* VBOX_WITH_64_BITS_GUESTS */
490
491	#undef PGM_SHW_TYPE
492	#undef PGM_SHW_NAME
493
494
495	/*
496	* Shadow - NEM / None.
497	*/
498	#define PGM_SHW_TYPE PGM_TYPE_NONE
499	#define PGM_SHW_NAME(name) PGM_SHW_NAME_NONE(name)
500	#include "PGMAllShw.h"
501
502	/* Guest - real mode */
503	#define PGM_GST_TYPE PGM_TYPE_REAL
504	#define PGM_GST_NAME(name) PGM_GST_NAME_REAL(name)
505	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_REAL(name)
506	#include "PGMGstDefs.h"
507	#include "PGMAllBth.h"
508	#undef PGM_BTH_NAME
509	#undef PGM_GST_TYPE
510	#undef PGM_GST_NAME
511
512	/* Guest - protected mode */
513	#define PGM_GST_TYPE PGM_TYPE_PROT
514	#define PGM_GST_NAME(name) PGM_GST_NAME_PROT(name)
515	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PROT(name)
516	#include "PGMGstDefs.h"
517	#include "PGMAllBth.h"
518	#undef PGM_BTH_NAME
519	#undef PGM_GST_TYPE
520	#undef PGM_GST_NAME
521
522	/* Guest - 32-bit mode */
523	#define PGM_GST_TYPE PGM_TYPE_32BIT
524	#define PGM_GST_NAME(name) PGM_GST_NAME_32BIT(name)
525	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_32BIT(name)
526	#include "PGMGstDefs.h"
527	#include "PGMAllBth.h"
528	#undef PGM_BTH_NAME
529	#undef PGM_GST_TYPE
530	#undef PGM_GST_NAME
531
532	/* Guest - PAE mode */
533	#define PGM_GST_TYPE PGM_TYPE_PAE
534	#define PGM_GST_NAME(name) PGM_GST_NAME_PAE(name)
535	#define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_PAE(name)
536	#include "PGMGstDefs.h"
537	#include "PGMAllBth.h"
538	#undef PGM_BTH_NAME
539	#undef PGM_GST_TYPE
540	#undef PGM_GST_NAME
541
542	#ifdef VBOX_WITH_64_BITS_GUESTS
543	/* Guest - AMD64 mode */
544	# define PGM_GST_TYPE PGM_TYPE_AMD64
545	# define PGM_GST_NAME(name) PGM_GST_NAME_AMD64(name)
546	# define PGM_BTH_NAME(name) PGM_BTH_NAME_NONE_AMD64(name)
547	# include "PGMGstDefs.h"
548	# include "PGMAllBth.h"
549	# undef PGM_BTH_NAME
550	# undef PGM_GST_TYPE
551	# undef PGM_GST_NAME
552	#endif /* VBOX_WITH_64_BITS_GUESTS */
553
554	#undef PGM_SHW_TYPE
555	#undef PGM_SHW_NAME
556
557
558
559	/**
560	* Guest mode data array.
561	*/
562	PGMMODEDATAGST const g_aPgmGuestModeData[PGM_GUEST_MODE_DATA_ARRAY_SIZE] =
563	{
564	{ UINT32_MAX, NULL, NULL, NULL, NULL }, /* 0 */
565	{
566	PGM_TYPE_REAL,
567	PGM_GST_NAME_REAL(GetPage),
568	PGM_GST_NAME_REAL(ModifyPage),
569	PGM_GST_NAME_REAL(Enter),
570	PGM_GST_NAME_REAL(Exit),
571	#ifdef IN_RING3
572	PGM_GST_NAME_REAL(Relocate),
573	#endif
574	},
575	{
576	PGM_TYPE_PROT,
577	PGM_GST_NAME_PROT(GetPage),
578	PGM_GST_NAME_PROT(ModifyPage),
579	PGM_GST_NAME_PROT(Enter),
580	PGM_GST_NAME_PROT(Exit),
581	#ifdef IN_RING3
582	PGM_GST_NAME_PROT(Relocate),
583	#endif
584	},
585	{
586	PGM_TYPE_32BIT,
587	PGM_GST_NAME_32BIT(GetPage),
588	PGM_GST_NAME_32BIT(ModifyPage),
589	PGM_GST_NAME_32BIT(Enter),
590	PGM_GST_NAME_32BIT(Exit),
591	#ifdef IN_RING3
592	PGM_GST_NAME_32BIT(Relocate),
593	#endif
594	},
595	{
596	PGM_TYPE_PAE,
597	PGM_GST_NAME_PAE(GetPage),
598	PGM_GST_NAME_PAE(ModifyPage),
599	PGM_GST_NAME_PAE(Enter),
600	PGM_GST_NAME_PAE(Exit),
601	#ifdef IN_RING3
602	PGM_GST_NAME_PAE(Relocate),
603	#endif
604	},
605	#ifdef VBOX_WITH_64_BITS_GUESTS
606	{
607	PGM_TYPE_AMD64,
608	PGM_GST_NAME_AMD64(GetPage),
609	PGM_GST_NAME_AMD64(ModifyPage),
610	PGM_GST_NAME_AMD64(Enter),
611	PGM_GST_NAME_AMD64(Exit),
612	# ifdef IN_RING3
613	PGM_GST_NAME_AMD64(Relocate),
614	# endif
615	},
616	#endif
617	};
618
619
620	/**
621	* The shadow mode data array.
622	*/
623	PGMMODEDATASHW const g_aPgmShadowModeData[PGM_SHADOW_MODE_DATA_ARRAY_SIZE] =
624	{
625	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* 0 */
626	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_REAL */
627	{ UINT8_MAX, NULL, NULL, NULL, NULL }, /* PGM_TYPE_PROT */
628	{
629	PGM_TYPE_32BIT,
630	PGM_SHW_NAME_32BIT(GetPage),
631	PGM_SHW_NAME_32BIT(ModifyPage),
632	PGM_SHW_NAME_32BIT(Enter),
633	PGM_SHW_NAME_32BIT(Exit),
634	#ifdef IN_RING3
635	PGM_SHW_NAME_32BIT(Relocate),
636	#endif
637	},
638	{
639	PGM_TYPE_PAE,
640	PGM_SHW_NAME_PAE(GetPage),
641	PGM_SHW_NAME_PAE(ModifyPage),
642	PGM_SHW_NAME_PAE(Enter),
643	PGM_SHW_NAME_PAE(Exit),
644	#ifdef IN_RING3
645	PGM_SHW_NAME_PAE(Relocate),
646	#endif
647	},
648	{
649	PGM_TYPE_AMD64,
650	PGM_SHW_NAME_AMD64(GetPage),
651	PGM_SHW_NAME_AMD64(ModifyPage),
652	PGM_SHW_NAME_AMD64(Enter),
653	PGM_SHW_NAME_AMD64(Exit),
654	#ifdef IN_RING3
655	PGM_SHW_NAME_AMD64(Relocate),
656	#endif
657	},
658	{
659	PGM_TYPE_NESTED_32BIT,
660	PGM_SHW_NAME_NESTED_32BIT(GetPage),
661	PGM_SHW_NAME_NESTED_32BIT(ModifyPage),
662	PGM_SHW_NAME_NESTED_32BIT(Enter),
663	PGM_SHW_NAME_NESTED_32BIT(Exit),
664	#ifdef IN_RING3
665	PGM_SHW_NAME_NESTED_32BIT(Relocate),
666	#endif
667	},
668	{
669	PGM_TYPE_NESTED_PAE,
670	PGM_SHW_NAME_NESTED_PAE(GetPage),
671	PGM_SHW_NAME_NESTED_PAE(ModifyPage),
672	PGM_SHW_NAME_NESTED_PAE(Enter),
673	PGM_SHW_NAME_NESTED_PAE(Exit),
674	#ifdef IN_RING3
675	PGM_SHW_NAME_NESTED_PAE(Relocate),
676	#endif
677	},
678	{
679	PGM_TYPE_NESTED_AMD64,
680	PGM_SHW_NAME_NESTED_AMD64(GetPage),
681	PGM_SHW_NAME_NESTED_AMD64(ModifyPage),
682	PGM_SHW_NAME_NESTED_AMD64(Enter),
683	PGM_SHW_NAME_NESTED_AMD64(Exit),
684	#ifdef IN_RING3
685	PGM_SHW_NAME_NESTED_AMD64(Relocate),
686	#endif
687	},
688	{
689	PGM_TYPE_EPT,
690	PGM_SHW_NAME_EPT(GetPage),
691	PGM_SHW_NAME_EPT(ModifyPage),
692	PGM_SHW_NAME_EPT(Enter),
693	PGM_SHW_NAME_EPT(Exit),
694	#ifdef IN_RING3
695	PGM_SHW_NAME_EPT(Relocate),
696	#endif
697	},
698	{
699	PGM_TYPE_NONE,
700	PGM_SHW_NAME_NONE(GetPage),
701	PGM_SHW_NAME_NONE(ModifyPage),
702	PGM_SHW_NAME_NONE(Enter),
703	PGM_SHW_NAME_NONE(Exit),
704	#ifdef IN_RING3
705	PGM_SHW_NAME_NONE(Relocate),
706	#endif
707	},
708	};
709
710
711	/**
712	* The guest+shadow mode data array.
713	*/
714	PGMMODEDATABTH const g_aPgmBothModeData[PGM_BOTH_MODE_DATA_ARRAY_SIZE] =
715	{
716	#if !defined(IN_RING3) && !defined(VBOX_STRICT)
717	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
718	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
719	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler) }
720
721	#elif !defined(IN_RING3) && defined(VBOX_STRICT)
722	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
723	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
724	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(Trap0eHandler), Nm(AssertCR3) }
725
726	#elif defined(IN_RING3) && !defined(VBOX_STRICT)
727	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL }
728	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
729	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), }
730
731	#elif defined(IN_RING3) && defined(VBOX_STRICT)
732	# define PGMMODEDATABTH_NULL_ENTRY() { UINT32_MAX, UINT32_MAX, NULL, NULL, NULL, NULL, NULL, NULL, NULL }
733	# define PGMMODEDATABTH_ENTRY(uShwT, uGstT, Nm) \
734	{ uShwT, uGstT, Nm(InvalidatePage), Nm(SyncCR3), Nm(PrefetchPage), Nm(VerifyAccessSyncPage), Nm(MapCR3), Nm(UnmapCR3), Nm(Enter), Nm(AssertCR3) }
735
736	#else
737	# error "Misconfig."
738	#endif
739
740	/* 32-bit shadow paging mode: */
741	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
742	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_32BIT_REAL),
743	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_32BIT_PROT),
744	PGMMODEDATABTH_ENTRY(PGM_TYPE_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_32BIT_32BIT),
745	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_PAE - illegal */
746	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_AMD64 - illegal */
747	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
748	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_PAE - illegal */
749	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
750	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_EPT - illegal */
751	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_32BIT, PGM_TYPE_NONE - illegal */
752
753	/* PAE shadow paging mode: */
754	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
755	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_PAE_REAL),
756	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_PAE_PROT),
757	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_PAE_32BIT),
758	PGMMODEDATABTH_ENTRY(PGM_TYPE_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_PAE_PAE),
759	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_AMD64 - illegal */
760	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_32BIT - illegal */
761	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_PAE - illegal */
762	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
763	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_EPT - illegal */
764	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_PAE, PGM_TYPE_NONE - illegal */
765
766	/* AMD64 shadow paging mode: */
767	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
768	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_AMD64_REAL),
769	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_AMD64_PROT),
770	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_AMD64_32BIT),
771	PGMMODEDATABTH_NULL_ENTRY(), //PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_AMD64_PAE),
772	#ifdef VBOX_WITH_64_BITS_GUESTS
773	PGMMODEDATABTH_ENTRY(PGM_TYPE_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_AMD64_AMD64),
774	#else
775	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_AMD64 - illegal */
776	#endif
777	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
778	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_PAE - illegal */
779	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
780	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_EPT - illegal */
781	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_AMD64, PGM_TYPE_NONE - illegal */
782
783	/* 32-bit nested paging mode: */
784	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
785	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_32BIT_REAL),
786	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_32BIT_PROT),
787	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_32BIT_32BIT),
788	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_32BIT_PAE),
789	#ifdef VBOX_WITH_64_BITS_GUESTS
790	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_32BIT_AMD64),
791	#else
792	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_AMD64 - illegal */
793	#endif
794	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_32BIT - illegal */
795	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_PAE - illegal */
796	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NESTED_AMD64 - illegal */
797	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_EPT - illegal */
798	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_32BIT, PGM_TYPE_NONE - illegal */
799
800	/* PAE nested paging mode: */
801	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
802	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_PAE_REAL),
803	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_PAE_PROT),
804	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_PAE_32BIT),
805	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_PAE_PAE),
806	#ifdef VBOX_WITH_64_BITS_GUESTS
807	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_PAE_AMD64),
808	#else
809	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_AMD64 - illegal */
810	#endif
811	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_32BIT - illegal */
812	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_PAE - illegal */
813	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NESTED_AMD64 - illegal */
814	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_EPT - illegal */
815	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_PAE, PGM_TYPE_NONE - illegal */
816
817	/* AMD64 nested paging mode: */
818	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
819	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_REAL, PGM_BTH_NAME_NESTED_AMD64_REAL),
820	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PROT, PGM_BTH_NAME_NESTED_AMD64_PROT),
821	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_32BIT, PGM_BTH_NAME_NESTED_AMD64_32BIT),
822	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_PAE, PGM_BTH_NAME_NESTED_AMD64_PAE),
823	#ifdef VBOX_WITH_64_BITS_GUESTS
824	PGMMODEDATABTH_ENTRY(PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64, PGM_BTH_NAME_NESTED_AMD64_AMD64),
825	#else
826	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_AMD64 - illegal */
827	#endif
828	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_32BIT - illegal */
829	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_PAE - illegal */
830	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NESTED_AMD64 - illegal */
831	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_EPT - illegal */
832	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NESTED_AMD64, PGM_TYPE_NONE - illegal */
833
834	/* EPT nested paging mode: */
835	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
836	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
837	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
838	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
839	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
840	#ifdef VBOX_WITH_64_BITS_GUESTS
841	PGMMODEDATABTH_ENTRY(PGM_TYPE_EPT, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
842	#else
843	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_AMD64 - illegal */
844	#endif
845	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_32BIT - illegal */
846	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_PAE - illegal */
847	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NESTED_AMD64 - illegal */
848	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_EPT - illegal */
849	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_EPT, PGM_TYPE_NONE - illegal */
850
851	/* NONE / NEM: */
852	PGMMODEDATABTH_NULL_ENTRY(), /* 0 */
853	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_REAL, PGM_BTH_NAME_EPT_REAL),
854	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PROT, PGM_BTH_NAME_EPT_PROT),
855	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_32BIT, PGM_BTH_NAME_EPT_32BIT),
856	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_PAE, PGM_BTH_NAME_EPT_PAE),
857	#ifdef VBOX_WITH_64_BITS_GUESTS
858	PGMMODEDATABTH_ENTRY(PGM_TYPE_NONE, PGM_TYPE_AMD64, PGM_BTH_NAME_EPT_AMD64),
859	#else
860	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_AMD64 - illegal */
861	#endif
862	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_32BIT - illegal */
863	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_PAE - illegal */
864	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NESTED_AMD64 - illegal */
865	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_EPT - illegal */
866	PGMMODEDATABTH_NULL_ENTRY(), /* PGM_TYPE_NONE, PGM_TYPE_NONE - illegal */
867
868
869	#undef PGMMODEDATABTH_ENTRY
870	#undef PGMMODEDATABTH_NULL_ENTRY
871	};
872
873
874	#ifdef IN_RING0
875	/**
876	* #PF Handler.
877	*
878	* @returns VBox status code (appropriate for trap handling and GC return).
879	* @param pVCpu The cross context virtual CPU structure.
880	* @param uErr The trap error code.
881	* @param pRegFrame Trap register frame.
882	* @param pvFault The fault address.
883	*/
884	VMMDECL(int) PGMTrap0eHandler(PVMCPUCC pVCpu, RTGCUINT uErr, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
885	{
886	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
887
888	Log(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv eip=%04x:%RGv cr3=%RGp\n", uErr, pvFault, pRegFrame->cs.Sel, (RTGCPTR)pRegFrame->rip, (RTGCPHYS)CPUMGetGuestCR3(pVCpu)));
889	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.StatRZTrap0e, a);
890	STAM_STATS({ pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = NULL; } );
891
892
893	# ifdef VBOX_WITH_STATISTICS
894	/*
895	* Error code stats.
896	*/
897	if (uErr & X86_TRAP_PF_US)
898	{
899	if (!(uErr & X86_TRAP_PF_P))
900	{
901	if (uErr & X86_TRAP_PF_RW)
902	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentWrite);
903	else
904	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNotPresentRead);
905	}
906	else if (uErr & X86_TRAP_PF_RW)
907	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSWrite);
908	else if (uErr & X86_TRAP_PF_RSVD)
909	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSReserved);
910	else if (uErr & X86_TRAP_PF_ID)
911	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSNXE);
912	else
913	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eUSRead);
914	}
915	else
916	{ /* Supervisor */
917	if (!(uErr & X86_TRAP_PF_P))
918	{
919	if (uErr & X86_TRAP_PF_RW)
920	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentWrite);
921	else
922	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVNotPresentRead);
923	}
924	else if (uErr & X86_TRAP_PF_RW)
925	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVWrite);
926	else if (uErr & X86_TRAP_PF_ID)
927	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSNXE);
928	else if (uErr & X86_TRAP_PF_RSVD)
929	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eSVReserved);
930	}
931	# endif /* VBOX_WITH_STATISTICS */
932
933	/*
934	* Call the worker.
935	*/
936	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
937	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
938	AssertReturn(g_aPgmBothModeData[idxBth].pfnTrap0eHandler, VERR_PGM_MODE_IPE);
939	bool fLockTaken = false;
940	int rc = g_aPgmBothModeData[idxBth].pfnTrap0eHandler(pVCpu, uErr, pRegFrame, pvFault, &fLockTaken);
941	if (fLockTaken)
942	{
943	PGM_LOCK_ASSERT_OWNER(pVM);
944	PGM_UNLOCK(pVM);
945	}
946	LogFlow(("PGMTrap0eHandler: uErr=%RGx pvFault=%RGv rc=%Rrc\n", uErr, pvFault, rc));
947
948	/*
949	* Return code tweaks.
950	*/
951	if (rc != VINF_SUCCESS)
952	{
953	if (rc == VINF_PGM_SYNCPAGE_MODIFIED_PDE)
954	rc = VINF_SUCCESS;
955
956	/* Note: hack alert for difficult to reproduce problem. */
957	if ( rc == VERR_PAGE_NOT_PRESENT /* SMP only ; disassembly might fail. */
958	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT /* seen with UNI & SMP */
959	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT /* seen with SMP */
960	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT) /* precaution */
961	{
962	Log(("WARNING: Unexpected VERR_PAGE_TABLE_NOT_PRESENT (%d) for page fault at %RGv error code %x (rip=%RGv)\n", rc, pvFault, uErr, pRegFrame->rip));
963	/* Some kind of inconsistency in the SMP case; it's safe to just execute the instruction again; not sure about single VCPU VMs though. */
964	rc = VINF_SUCCESS;
965	}
966	}
967
968	STAM_STATS({ if (rc == VINF_EM_RAW_GUEST_TRAP) STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZTrap0eGuestPF); });
969	STAM_STATS({ if (!pVCpu->pgmr0.s.pStatTrap0eAttributionR0)
970	pVCpu->pgmr0.s.pStatTrap0eAttributionR0 = &pVCpu->pgm.s.Stats.StatRZTrap0eTime2Misc; });
971	STAM_PROFILE_STOP_EX(&pVCpu->pgm.s.Stats.StatRZTrap0e, pVCpu->pgmr0.s.pStatTrap0eAttributionR0, a);
972	return rc;
973	}
974	#endif /* IN_RING0 */
975
976
977	/**
978	* Prefetch a page
979	*
980	* Typically used to sync commonly used pages before entering raw mode
981	* after a CR3 reload.
982	*
983	* @returns VBox status code suitable for scheduling.
984	* @retval VINF_SUCCESS on success.
985	* @retval VINF_PGM_SYNC_CR3 if we're out of shadow pages or something like that.
986	* @param pVCpu The cross context virtual CPU structure.
987	* @param GCPtrPage Page to invalidate.
988	*/
989	VMMDECL(int) PGMPrefetchPage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
990	{
991	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
992
993	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
994	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
995	AssertReturn(g_aPgmBothModeData[idxBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
996	int rc = g_aPgmBothModeData[idxBth].pfnPrefetchPage(pVCpu, GCPtrPage);
997
998	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,Prefetch), a);
999	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
1000	return rc;
1001	}
1002
1003
1004	/**
1005	* Verifies a range of pages for read or write access
1006	*
1007	* Only checks the guest's page tables
1008	*
1009	* @returns VBox status code.
1010	* @param pVCpu The cross context virtual CPU structure.
1011	* @param Addr Guest virtual address to check
1012	* @param cbSize Access size
1013	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1014	* @remarks Current not in use.
1015	*/
1016	VMMDECL(int) PGMIsValidAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1017	{
1018	/*
1019	* Validate input.
1020	*/
1021	if (fAccess & ~(X86_PTE_US \| X86_PTE_RW))
1022	{
1023	AssertMsgFailed(("PGMIsValidAccess: invalid access type %08x\n", fAccess));
1024	return VERR_INVALID_PARAMETER;
1025	}
1026
1027	uint64_t fPage;
1028	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPage, NULL);
1029	if (RT_FAILURE(rc))
1030	{
1031	Log(("PGMIsValidAccess: access violation for %RGv rc=%d\n", Addr, rc));
1032	return VINF_EM_RAW_GUEST_TRAP;
1033	}
1034
1035	/*
1036	* Check if the access would cause a page fault
1037	*
1038	* Note that hypervisor page directories are not present in the guest's tables, so this check
1039	* is sufficient.
1040	*/
1041	bool fWrite = !!(fAccess & X86_PTE_RW);
1042	bool fUser = !!(fAccess & X86_PTE_US);
1043	if ( !(fPage & X86_PTE_P)
1044	\|\| (fWrite && !(fPage & X86_PTE_RW))
1045	\|\| (fUser && !(fPage & X86_PTE_US)) )
1046	{
1047	Log(("PGMIsValidAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPage, fWrite, fUser));
1048	return VINF_EM_RAW_GUEST_TRAP;
1049	}
1050	if ( RT_SUCCESS(rc)
1051	&& PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize))
1052	return PGMIsValidAccess(pVCpu, Addr + PAGE_SIZE, (cbSize > PAGE_SIZE) ? cbSize - PAGE_SIZE : 1, fAccess);
1053	return rc;
1054	}
1055
1056
1057	/**
1058	* Verifies a range of pages for read or write access
1059	*
1060	* Supports handling of pages marked for dirty bit tracking and CSAM
1061	*
1062	* @returns VBox status code.
1063	* @param pVCpu The cross context virtual CPU structure.
1064	* @param Addr Guest virtual address to check
1065	* @param cbSize Access size
1066	* @param fAccess Access type (r/w, user/supervisor (X86_PTE_*))
1067	*/
1068	VMMDECL(int) PGMVerifyAccess(PVMCPUCC pVCpu, RTGCPTR Addr, uint32_t cbSize, uint32_t fAccess)
1069	{
1070	PVM pVM = pVCpu->CTX_SUFF(pVM);
1071
1072	AssertMsg(!(fAccess & ~(X86_PTE_US \| X86_PTE_RW)), ("PGMVerifyAccess: invalid access type %08x\n", fAccess));
1073
1074	/*
1075	* Get going.
1076	*/
1077	uint64_t fPageGst;
1078	int rc = PGMGstGetPage(pVCpu, (RTGCPTR)Addr, &fPageGst, NULL);
1079	if (RT_FAILURE(rc))
1080	{
1081	Log(("PGMVerifyAccess: access violation for %RGv rc=%d\n", Addr, rc));
1082	return VINF_EM_RAW_GUEST_TRAP;
1083	}
1084
1085	/*
1086	* Check if the access would cause a page fault
1087	*
1088	* Note that hypervisor page directories are not present in the guest's tables, so this check
1089	* is sufficient.
1090	*/
1091	const bool fWrite = !!(fAccess & X86_PTE_RW);
1092	const bool fUser = !!(fAccess & X86_PTE_US);
1093	if ( !(fPageGst & X86_PTE_P)
1094	\|\| (fWrite && !(fPageGst & X86_PTE_RW))
1095	\|\| (fUser && !(fPageGst & X86_PTE_US)) )
1096	{
1097	Log(("PGMVerifyAccess: access violation for %RGv attr %#llx vs %d:%d\n", Addr, fPageGst, fWrite, fUser));
1098	return VINF_EM_RAW_GUEST_TRAP;
1099	}
1100
1101	if (!pVM->pgm.s.fNestedPaging)
1102	{
1103	/*
1104	* Next step is to verify if we protected this page for dirty bit tracking or for CSAM scanning
1105	*/
1106	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, NULL, NULL);
1107	if ( rc == VERR_PAGE_NOT_PRESENT
1108	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT)
1109	{
1110	/*
1111	* Page is not present in our page tables.
1112	* Try to sync it!
1113	*/
1114	Assert(X86_TRAP_PF_RW == X86_PTE_RW && X86_TRAP_PF_US == X86_PTE_US);
1115	uint32_t const uErr = fAccess & (X86_TRAP_PF_RW \| X86_TRAP_PF_US);
1116	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1117	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
1118	AssertReturn(g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
1119	rc = g_aPgmBothModeData[idxBth].pfnVerifyAccessSyncPage(pVCpu, Addr, fPageGst, uErr);
1120	if (rc != VINF_SUCCESS)
1121	return rc;
1122	}
1123	else
1124	AssertMsg(rc == VINF_SUCCESS, ("PGMShwGetPage %RGv failed with %Rrc\n", Addr, rc));
1125	}
1126
1127	#if 0 /* def VBOX_STRICT; triggers too often now */
1128	/*
1129	* This check is a bit paranoid, but useful.
1130	*/
1131	/* Note! This will assert when writing to monitored pages (a bit annoying actually). */
1132	uint64_t fPageShw;
1133	rc = PGMShwGetPage(pVCpu, (RTGCPTR)Addr, &fPageShw, NULL);
1134	if ( (rc == VERR_PAGE_NOT_PRESENT \|\| RT_FAILURE(rc))
1135	\|\| (fWrite && !(fPageShw & X86_PTE_RW))
1136	\|\| (fUser && !(fPageShw & X86_PTE_US)) )
1137	{
1138	AssertMsgFailed(("Unexpected access violation for %RGv! rc=%Rrc write=%d user=%d\n",
1139	Addr, rc, fWrite && !(fPageShw & X86_PTE_RW), fUser && !(fPageShw & X86_PTE_US)));
1140	return VINF_EM_RAW_GUEST_TRAP;
1141	}
1142	#endif
1143
1144	if ( RT_SUCCESS(rc)
1145	&& ( PAGE_ADDRESS(Addr) != PAGE_ADDRESS(Addr + cbSize - 1)
1146	\|\| Addr + cbSize < Addr))
1147	{
1148	/* Don't recursively call PGMVerifyAccess as we might run out of stack. */
1149	for (;;)
1150	{
1151	Addr += PAGE_SIZE;
1152	if (cbSize > PAGE_SIZE)
1153	cbSize -= PAGE_SIZE;
1154	else
1155	cbSize = 1;
1156	rc = PGMVerifyAccess(pVCpu, Addr, 1, fAccess);
1157	if (rc != VINF_SUCCESS)
1158	break;
1159	if (PAGE_ADDRESS(Addr) == PAGE_ADDRESS(Addr + cbSize - 1))
1160	break;
1161	}
1162	}
1163	return rc;
1164	}
1165
1166
1167	/**
1168	* Emulation of the invlpg instruction (HC only actually).
1169	*
1170	* @returns Strict VBox status code, special care required.
1171	* @retval VINF_PGM_SYNC_CR3 - handled.
1172	* @retval VINF_EM_RAW_EMULATE_INSTR - not handled (RC only).
1173	* @retval VERR_REM_FLUSHED_PAGES_OVERFLOW - not handled.
1174	*
1175	* @param pVCpu The cross context virtual CPU structure.
1176	* @param GCPtrPage Page to invalidate.
1177	*
1178	* @remark ASSUMES the page table entry or page directory is valid. Fairly
1179	* safe, but there could be edge cases!
1180	*
1181	* @todo Flush page or page directory only if necessary!
1182	* @todo VBOXSTRICTRC
1183	*/
1184	VMMDECL(int) PGMInvalidatePage(PVMCPUCC pVCpu, RTGCPTR GCPtrPage)
1185	{
1186	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1187	int rc;
1188	Log3(("PGMInvalidatePage: GCPtrPage=%RGv\n", GCPtrPage));
1189
1190	IEMTlbInvalidatePage(pVCpu, GCPtrPage);
1191
1192	/*
1193	* Call paging mode specific worker.
1194	*/
1195	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1196	PGM_LOCK_VOID(pVM);
1197
1198	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
1199	AssertReturnStmt(idxBth < RT_ELEMENTS(g_aPgmBothModeData), PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1200	AssertReturnStmt(g_aPgmBothModeData[idxBth].pfnInvalidatePage, PGM_UNLOCK(pVM), VERR_PGM_MODE_IPE);
1201	rc = g_aPgmBothModeData[idxBth].pfnInvalidatePage(pVCpu, GCPtrPage);
1202
1203	PGM_UNLOCK(pVM);
1204	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,InvalidatePage), a);
1205
1206	/* Ignore all irrelevant error codes. */
1207	if ( rc == VERR_PAGE_NOT_PRESENT
1208	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
1209	\|\| rc == VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT
1210	\|\| rc == VERR_PAGE_MAP_LEVEL4_NOT_PRESENT)
1211	rc = VINF_SUCCESS;
1212
1213	return rc;
1214	}
1215
1216
1217	/**
1218	* Executes an instruction using the interpreter.
1219	*
1220	* @returns VBox status code (appropriate for trap handling and GC return).
1221	* @param pVM The cross context VM structure.
1222	* @param pVCpu The cross context virtual CPU structure.
1223	* @param pRegFrame Register frame.
1224	* @param pvFault Fault address.
1225	*/
1226	VMMDECL(VBOXSTRICTRC) PGMInterpretInstruction(PVMCC pVM, PVMCPUCC pVCpu, PCPUMCTXCORE pRegFrame, RTGCPTR pvFault)
1227	{
1228	NOREF(pVM);
1229	VBOXSTRICTRC rc = EMInterpretInstruction(pVCpu, pRegFrame, pvFault);
1230	if (rc == VERR_EM_INTERPRETER)
1231	rc = VINF_EM_RAW_EMULATE_INSTR;
1232	if (rc != VINF_SUCCESS)
1233	Log(("PGMInterpretInstruction: returns %Rrc (pvFault=%RGv)\n", VBOXSTRICTRC_VAL(rc), pvFault));
1234	return rc;
1235	}
1236
1237
1238	/**
1239	* Gets effective page information (from the VMM page directory).
1240	*
1241	* @returns VBox status code.
1242	* @param pVCpu The cross context virtual CPU structure.
1243	* @param GCPtr Guest Context virtual address of the page.
1244	* @param pfFlags Where to store the flags. These are X86_PTE_*.
1245	* @param pHCPhys Where to store the HC physical address of the page.
1246	* This is page aligned.
1247	* @remark You should use PGMMapGetPage() for pages in a mapping.
1248	*/
1249	VMMDECL(int) PGMShwGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTHCPHYS pHCPhys)
1250	{
1251	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1252	PGM_LOCK_VOID(pVM);
1253
1254	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1255	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1256	AssertReturn(g_aPgmShadowModeData[idxShw].pfnGetPage, VERR_PGM_MODE_IPE);
1257	int rc = g_aPgmShadowModeData[idxShw].pfnGetPage(pVCpu, GCPtr, pfFlags, pHCPhys);
1258
1259	PGM_UNLOCK(pVM);
1260	return rc;
1261	}
1262
1263
1264	/**
1265	* Modify page flags for a range of pages in the shadow context.
1266	*
1267	* The existing flags are ANDed with the fMask and ORed with the fFlags.
1268	*
1269	* @returns VBox status code.
1270	* @param pVCpu The cross context virtual CPU structure.
1271	* @param GCPtr Virtual address of the first page in the range.
1272	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
1273	* @param fMask The AND mask - page flags X86_PTE_*.
1274	* Be very CAREFUL when ~'ing constants which could be 32-bit!
1275	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1276	* @remark You must use PGMMapModifyPage() for pages in a mapping.
1277	*/
1278	DECLINLINE(int) pdmShwModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t fFlags, uint64_t fMask, uint32_t fOpFlags)
1279	{
1280	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
1281	Assert(!(fOpFlags & ~(PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT)));
1282
1283	GCPtr &= PAGE_BASE_GC_MASK; /** @todo this ain't necessary, right... */
1284
1285	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1286	PGM_LOCK_VOID(pVM);
1287
1288	uintptr_t idxShw = pVCpu->pgm.s.idxShadowModeData;
1289	AssertReturn(idxShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
1290	AssertReturn(g_aPgmShadowModeData[idxShw].pfnModifyPage, VERR_PGM_MODE_IPE);
1291	int rc = g_aPgmShadowModeData[idxShw].pfnModifyPage(pVCpu, GCPtr, PAGE_SIZE, fFlags, fMask, fOpFlags);
1292
1293	PGM_UNLOCK(pVM);
1294	return rc;
1295	}
1296
1297
1298	/**
1299	* Changing the page flags for a single page in the shadow page tables so as to
1300	* make it read-only.
1301	*
1302	* @returns VBox status code.
1303	* @param pVCpu The cross context virtual CPU structure.
1304	* @param GCPtr Virtual address of the first page in the range.
1305	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1306	*/
1307	VMMDECL(int) PGMShwMakePageReadonly(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1308	{
1309	return pdmShwModifyPage(pVCpu, GCPtr, 0, ~(uint64_t)X86_PTE_RW, fOpFlags);
1310	}
1311
1312
1313	/**
1314	* Changing the page flags for a single page in the shadow page tables so as to
1315	* make it writable.
1316	*
1317	* The call must know with 101% certainty that the guest page tables maps this
1318	* as writable too. This function will deal shared, zero and write monitored
1319	* pages.
1320	*
1321	* @returns VBox status code.
1322	* @param pVCpu The cross context virtual CPU structure.
1323	* @param GCPtr Virtual address of the first page in the range.
1324	* @param fOpFlags A combination of the PGM_MK_PK_XXX flags.
1325	*/
1326	VMMDECL(int) PGMShwMakePageWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1327	{
1328	if (pVCpu->pgm.s.enmShadowMode != PGMMODE_NONE) /* avoid assertions */
1329	return pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)0, fOpFlags);
1330	return VINF_SUCCESS;
1331	}
1332
1333
1334	/**
1335	* Changing the page flags for a single page in the shadow page tables so as to
1336	* make it not present.
1337	*
1338	* @returns VBox status code.
1339	* @param pVCpu The cross context virtual CPU structure.
1340	* @param GCPtr Virtual address of the first page in the range.
1341	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1342	*/
1343	VMMDECL(int) PGMShwMakePageNotPresent(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint32_t fOpFlags)
1344	{
1345	return pdmShwModifyPage(pVCpu, GCPtr, 0, 0, fOpFlags);
1346	}
1347
1348
1349	/**
1350	* Changing the page flags for a single page in the shadow page tables so as to
1351	* make it supervisor and writable.
1352	*
1353	* This if for dealing with CR0.WP=0 and readonly user pages.
1354	*
1355	* @returns VBox status code.
1356	* @param pVCpu The cross context virtual CPU structure.
1357	* @param GCPtr Virtual address of the first page in the range.
1358	* @param fBigPage Whether or not this is a big page. If it is, we have to
1359	* change the shadow PDE as well. If it isn't, the caller
1360	* has checked that the shadow PDE doesn't need changing.
1361	* We ASSUME 4KB pages backing the big page here!
1362	* @param fOpFlags A combination of the PGM_MK_PG_XXX flags.
1363	*/
1364	int pgmShwMakePageSupervisorAndWritable(PVMCPUCC pVCpu, RTGCPTR GCPtr, bool fBigPage, uint32_t fOpFlags)
1365	{
1366	int rc = pdmShwModifyPage(pVCpu, GCPtr, X86_PTE_RW, ~(uint64_t)X86_PTE_US, fOpFlags);
1367	if (rc == VINF_SUCCESS && fBigPage)
1368	{
1369	/* this is a bit ugly... */
1370	switch (pVCpu->pgm.s.enmShadowMode)
1371	{
1372	case PGMMODE_32_BIT:
1373	{
1374	PX86PDE pPde = pgmShwGet32BitPDEPtr(pVCpu, GCPtr);
1375	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1376	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1377	pPde->u \|= X86_PDE_RW;
1378	Log(("-> PDE=%#llx (32)\n", pPde->u));
1379	break;
1380	}
1381	case PGMMODE_PAE:
1382	case PGMMODE_PAE_NX:
1383	{
1384	PX86PDEPAE pPde = pgmShwGetPaePDEPtr(pVCpu, GCPtr);
1385	AssertReturn(pPde, VERR_INTERNAL_ERROR_3);
1386	Log(("pgmShwMakePageSupervisorAndWritable: PDE=%#llx", pPde->u));
1387	pPde->u \|= X86_PDE_RW;
1388	Log(("-> PDE=%#llx (PAE)\n", pPde->u));
1389	break;
1390	}
1391	default:
1392	AssertFailedReturn(VERR_INTERNAL_ERROR_4);
1393	}
1394	}
1395	return rc;
1396	}
1397
1398
1399	/**
1400	* Gets the shadow page directory for the specified address, PAE.
1401	*
1402	* @returns Pointer to the shadow PD.
1403	* @param pVCpu The cross context virtual CPU structure.
1404	* @param GCPtr The address.
1405	* @param uGstPdpe Guest PDPT entry. Valid.
1406	* @param ppPD Receives address of page directory
1407	*/
1408	int pgmShwSyncPaePDPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1409	{
1410	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1411	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1412	PPGMPOOLPAGE pShwPage;
1413	int rc;
1414	PGM_LOCK_ASSERT_OWNER(pVM);
1415
1416
1417	/* Allocate page directory if not present. */
1418	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1419	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1420	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1421	X86PGPAEUINT const uPdpe = pPdpe->u;
1422	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1423	{
1424	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1425	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1426	Assert((pPdpe->u & X86_PDPE_PG_MASK) == pShwPage->Core.Key);
1427
1428	pgmPoolCacheUsed(pPool, pShwPage);
1429
1430	/* Update the entry if necessary. */
1431	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS);
1432	if (uPdpeNew == uPdpe)
1433	{ /* likely */ }
1434	else
1435	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1436	}
1437	else
1438	{
1439	RTGCPTR64 GCPdPt;
1440	PGMPOOLKIND enmKind;
1441	if (pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu))
1442	{
1443	/* AMD-V nested paging or real/protected mode without paging. */
1444	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1445	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1446	}
1447	else
1448	{
1449	if (CPUMGetGuestCR4(pVCpu) & X86_CR4_PAE)
1450	{
1451	if (!(uGstPdpe & X86_PDPE_P))
1452	{
1453	/* PD not present; guest must reload CR3 to change it.
1454	* No need to monitor anything in this case. */
1455	Assert(VM_IS_RAW_MODE_ENABLED(pVM));
1456	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1457	enmKind = PGMPOOLKIND_PAE_PD_PHYS;
1458	Assert(uGstPdpe & X86_PDPE_P); /* caller should do this already */
1459	}
1460	else
1461	{
1462	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1463	enmKind = PGMPOOLKIND_PAE_PD_FOR_PAE_PD;
1464	}
1465	}
1466	else
1467	{
1468	GCPdPt = CPUMGetGuestCR3(pVCpu);
1469	enmKind = (PGMPOOLKIND)(PGMPOOLKIND_PAE_PD0_FOR_32BIT_PD + iPdPt);
1470	}
1471	}
1472
1473	/* Create a reference back to the PDPT by using the index in its shadow page. */
1474	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1475	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPdPt, false /fLockPage/,
1476	&pShwPage);
1477	AssertRCReturn(rc, rc);
1478
1479	/* Hook it up. */
1480	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| (uGstPdpe & (X86_PDPE_P \| X86_PDPE_A)) \| (uPdpe & PGM_PDPT_FLAGS));
1481	}
1482	PGM_DYNMAP_UNUSED_HINT(pVCpu, pPdpe);
1483
1484	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1485	return VINF_SUCCESS;
1486	}
1487
1488
1489	/**
1490	* Gets the pointer to the shadow page directory entry for an address, PAE.
1491	*
1492	* @returns Pointer to the PDE.
1493	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1494	* @param GCPtr The address.
1495	* @param ppShwPde Receives the address of the pgm pool page for the shadow page directory
1496	*/
1497	DECLINLINE(int) pgmShwGetPaePoolPagePD(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPOOLPAGE *ppShwPde)
1498	{
1499	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1500	PGM_LOCK_ASSERT_OWNER(pVM);
1501
1502	PX86PDPT pPdpt = pgmShwGetPaePDPTPtr(pVCpu);
1503	AssertReturn(pPdpt, VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT); /* can't happen */
1504	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_PAE;
1505	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1506	if (!(uPdpe & X86_PDPE_P))
1507	{
1508	LogFlow(("pgmShwGetPaePoolPagePD: PD %d not present (%RX64)\n", iPdPt, uPdpe));
1509	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1510	}
1511	AssertMsg(uPdpe & X86_PDPE_PG_MASK, ("GCPtr=%RGv\n", GCPtr));
1512
1513	/* Fetch the pgm pool shadow descriptor. */
1514	PPGMPOOLPAGE pShwPde = pgmPoolGetPage(pVM->pgm.s.CTX_SUFF(pPool), uPdpe & X86_PDPE_PG_MASK);
1515	AssertReturn(pShwPde, VERR_PGM_POOL_GET_PAGE_FAILED);
1516
1517	*ppShwPde = pShwPde;
1518	return VINF_SUCCESS;
1519	}
1520
1521
1522	/**
1523	* Syncs the SHADOW page directory pointer for the specified address.
1524	*
1525	* Allocates backing pages in case the PDPT or PML4 entry is missing.
1526	*
1527	* The caller is responsible for making sure the guest has a valid PD before
1528	* calling this function.
1529	*
1530	* @returns VBox status code.
1531	* @param pVCpu The cross context virtual CPU structure.
1532	* @param GCPtr The address.
1533	* @param uGstPml4e Guest PML4 entry (valid).
1534	* @param uGstPdpe Guest PDPT entry (valid).
1535	* @param ppPD Receives address of page directory
1536	*/
1537	static int pgmShwSyncLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, X86PGPAEUINT uGstPml4e, X86PGPAEUINT uGstPdpe, PX86PDPAE *ppPD)
1538	{
1539	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1540	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1541	bool const fNestedPagingOrNoGstPaging = pVM->pgm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabled(pVCpu);
1542	int rc;
1543
1544	PGM_LOCK_ASSERT_OWNER(pVM);
1545
1546	/*
1547	* PML4.
1548	*/
1549	PPGMPOOLPAGE pShwPage;
1550	{
1551	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1552	PX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1553	X86PGPAEUINT const uPml4e = pPml4e->u;
1554
1555	/* Allocate page directory pointer table if not present. */
1556	if (uPml4e & (X86_PML4E_P \| X86_PML4E_PG_MASK))
1557	{
1558	pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1559	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1560
1561	pgmPoolCacheUsed(pPool, pShwPage);
1562
1563	/* Update the entry if needed. */
1564	X86PGPAEUINT const uPml4eNew = pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1565	\| (uPml4e & PGM_PML4_FLAGS);
1566	if (uPml4e == uPml4eNew)
1567	{ /* likely */ }
1568	else
1569	ASMAtomicWriteU64(&pPml4e->u, uPml4eNew);
1570	}
1571	else
1572	{
1573	Assert(pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1574
1575	RTGCPTR64 GCPml4;
1576	PGMPOOLKIND enmKind;
1577	if (fNestedPagingOrNoGstPaging)
1578	{
1579	/* AMD-V nested paging or real/protected mode without paging */
1580	GCPml4 = (RTGCPTR64)iPml4 << X86_PML4_SHIFT;
1581	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_PHYS;
1582	}
1583	else
1584	{
1585	GCPml4 = uGstPml4e & X86_PML4E_PG_MASK;
1586	enmKind = PGMPOOLKIND_64BIT_PDPT_FOR_64BIT_PDPT;
1587	}
1588
1589	/* Create a reference back to the PDPT by using the index in its shadow page. */
1590	rc = pgmPoolAlloc(pVM, GCPml4, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1591	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1592	&pShwPage);
1593	AssertRCReturn(rc, rc);
1594
1595	/* Hook it up. */
1596	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| (uGstPml4e & pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask)
1597	\| (uPml4e & PGM_PML4_FLAGS));
1598	}
1599	}
1600
1601	/*
1602	* PDPT.
1603	*/
1604	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1605	PX86PDPT pPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1606	PX86PDPE pPdpe = &pPdpt->a[iPdPt];
1607	X86PGPAEUINT const uPdpe = pPdpe->u;
1608
1609	/* Allocate page directory if not present. */
1610	if (uPdpe & (X86_PDPE_P \| X86_PDPE_PG_MASK))
1611	{
1612	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1613	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1614
1615	pgmPoolCacheUsed(pPool, pShwPage);
1616
1617	/* Update the entry if needed. */
1618	X86PGPAEUINT const uPdpeNew = pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask)
1619	\| (uPdpe & PGM_PDPT_FLAGS);
1620	if (uPdpe == uPdpeNew)
1621	{ /* likely */ }
1622	else
1623	ASMAtomicWriteU64(&pPdpe->u, uPdpeNew);
1624	}
1625	else
1626	{
1627	RTGCPTR64 GCPdPt;
1628	PGMPOOLKIND enmKind;
1629	if (fNestedPagingOrNoGstPaging)
1630	{
1631	/* AMD-V nested paging or real/protected mode without paging */
1632	GCPdPt = (RTGCPTR64)iPdPt << X86_PDPT_SHIFT;
1633	enmKind = PGMPOOLKIND_64BIT_PD_FOR_PHYS;
1634	}
1635	else
1636	{
1637	GCPdPt = uGstPdpe & X86_PDPE_PG_MASK;
1638	enmKind = PGMPOOLKIND_64BIT_PD_FOR_64BIT_PD;
1639	}
1640
1641	/* Create a reference back to the PDPT by using the index in its shadow page. */
1642	rc = pgmPoolAlloc(pVM, GCPdPt, enmKind, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1643	pShwPage->idx, iPdPt, false /fLockPage/,
1644	&pShwPage);
1645	AssertRCReturn(rc, rc);
1646
1647	/* Hook it up. */
1648	ASMAtomicWriteU64(&pPdpe->u,
1649	pShwPage->Core.Key \| (uGstPdpe & pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask) \| (uPdpe & PGM_PDPT_FLAGS));
1650	}
1651
1652	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1653	return VINF_SUCCESS;
1654	}
1655
1656
1657	/**
1658	* Gets the SHADOW page directory pointer for the specified address (long mode).
1659	*
1660	* @returns VBox status code.
1661	* @param pVCpu The cross context virtual CPU structure.
1662	* @param GCPtr The address.
1663	* @param ppPml4e Receives the address of the page map level 4 entry.
1664	* @param ppPdpt Receives the address of the page directory pointer table.
1665	* @param ppPD Receives the address of the page directory.
1666	*/
1667	DECLINLINE(int) pgmShwGetLongModePDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PX86PML4E ppPml4e, PX86PDPT ppPdpt, PX86PDPAE *ppPD)
1668	{
1669	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1670	PGM_LOCK_ASSERT_OWNER(pVM);
1671
1672	/*
1673	* PML4
1674	*/
1675	const unsigned iPml4 = (GCPtr >> X86_PML4_SHIFT) & X86_PML4_MASK;
1676	PCX86PML4E pPml4e = pgmShwGetLongModePML4EPtr(pVCpu, iPml4);
1677	AssertReturn(pPml4e, VERR_PGM_PML4_MAPPING);
1678	if (ppPml4e)
1679	*ppPml4e = (PX86PML4E)pPml4e;
1680	X86PGPAEUINT const uPml4e = pPml4e->u;
1681	Log4(("pgmShwGetLongModePDPtr %RGv (%RHv) %RX64\n", GCPtr, pPml4e, uPml4e));
1682	if (!(uPml4e & X86_PML4E_P)) /** @todo other code is check for NULL page frame number! */
1683	return VERR_PAGE_MAP_LEVEL4_NOT_PRESENT;
1684
1685	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1686	PPGMPOOLPAGE pShwPage = pgmPoolGetPage(pPool, uPml4e & X86_PML4E_PG_MASK);
1687	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1688
1689	/*
1690	* PDPT
1691	*/
1692	const unsigned iPdPt = (GCPtr >> X86_PDPT_SHIFT) & X86_PDPT_MASK_AMD64;
1693	PCX86PDPT pPdpt = *ppPdpt = (PX86PDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1694	X86PGPAEUINT const uPdpe = pPdpt->a[iPdPt].u;
1695	if (!(uPdpe & X86_PDPE_P)) /** @todo other code is check for NULL page frame number! */
1696	return VERR_PAGE_DIRECTORY_PTR_NOT_PRESENT;
1697
1698	pShwPage = pgmPoolGetPage(pPool, uPdpe & X86_PDPE_PG_MASK);
1699	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1700
1701	*ppPD = (PX86PDPAE)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1702	Log4(("pgmShwGetLongModePDPtr %RGv -> ppPD=%p PDE=%p/%RX64\n", GCPtr, ppPD, &(ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK], (ppPD)->a[(GCPtr >> X86_PD_PAE_SHIFT) & X86_PD_PAE_MASK].u));
1703	return VINF_SUCCESS;
1704	}
1705
1706
1707	/**
1708	* Syncs the SHADOW EPT page directory pointer for the specified address. Allocates
1709	* backing pages in case the PDPT or PML4 entry is missing.
1710	*
1711	* @returns VBox status code.
1712	* @param pVCpu The cross context virtual CPU structure.
1713	* @param GCPtr The address.
1714	* @param ppPdpt Receives address of pdpt
1715	* @param ppPD Receives address of page directory
1716	*/
1717	static int pgmShwGetEPTPDPtr(PVMCPUCC pVCpu, RTGCPTR64 GCPtr, PEPTPDPT ppPdpt, PEPTPD ppPD)
1718	{
1719	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
1720	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
1721	int rc;
1722
1723	Assert(pVM->pgm.s.fNestedPaging);
1724	PGM_LOCK_ASSERT_OWNER(pVM);
1725
1726	/*
1727	* PML4 level.
1728	*/
1729
1730	PEPTPML4 pPml4 = (PEPTPML4)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pVCpu->pgm.s.CTX_SUFF(pShwPageCR3));
1731	Assert(pPml4);
1732
1733	/* Allocate page directory pointer table if not present. */
1734	PPGMPOOLPAGE pShwPage;
1735	{
1736	const unsigned iPml4 = (GCPtr >> EPT_PML4_SHIFT) & EPT_PML4_MASK;
1737	PEPTPML4E pPml4e = &pPml4->a[iPml4];
1738	EPTPML4E Pml4e;
1739	Pml4e.u = pPml4e->u;
1740	if (!(Pml4e.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1741	{
1742	RTGCPTR64 GCPml4 = (RTGCPTR64)iPml4 << EPT_PML4_SHIFT;
1743
1744	rc = pgmPoolAlloc(pVM, GCPml4, PGMPOOLKIND_EPT_PDPT_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1745	pVCpu->pgm.s.CTX_SUFF(pShwPageCR3)->idx, iPml4, false /fLockPage/,
1746	&pShwPage);
1747	AssertRCReturn(rc, rc);
1748
1749	/* Hook up the new PDPT now. */
1750	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1751	}
1752	else
1753	{
1754	pShwPage = pgmPoolGetPage(pPool, pPml4e->u & EPT_PML4E_PG_MASK);
1755	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1756
1757	pgmPoolCacheUsed(pPool, pShwPage);
1758
1759	/* Hook up the cached PDPT if needed (probably not given 512512 PTs to sync). /
1760	if (Pml4e.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1761	{ }
1762	else
1763	ASMAtomicWriteU64(&pPml4e->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1764	}
1765	}
1766
1767	/*
1768	* PDPT level.
1769	*/
1770	const unsigned iPdPt = (GCPtr >> EPT_PDPT_SHIFT) & EPT_PDPT_MASK;
1771	PEPTPDPT pPdpt = (PEPTPDPT)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1772	PEPTPDPTE pPdpe = &pPdpt->a[iPdPt];
1773
1774	if (ppPdpt)
1775	*ppPdpt = pPdpt;
1776
1777	/* Allocate page directory if not present. */
1778	EPTPDPTE Pdpe;
1779	Pdpe.u = pPdpe->u;
1780	if (!(Pdpe.u & (EPT_E_PG_MASK \| EPT_E_READ)))
1781	{
1782	RTGCPTR64 GCPdPt = (RTGCPTR64)iPdPt << EPT_PDPT_SHIFT;
1783	rc = pgmPoolAlloc(pVM, GCPdPt, PGMPOOLKIND_EPT_PD_FOR_PHYS, PGMPOOLACCESS_DONTCARE, PGM_A20_IS_ENABLED(pVCpu),
1784	pShwPage->idx, iPdPt, false /fLockPage/,
1785	&pShwPage);
1786	AssertRCReturn(rc, rc);
1787
1788	/* Hook up the new PD now. */
1789	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1790	}
1791	else
1792	{
1793	pShwPage = pgmPoolGetPage(pPool, pPdpe->u & EPT_PDPTE_PG_MASK);
1794	AssertReturn(pShwPage, VERR_PGM_POOL_GET_PAGE_FAILED);
1795
1796	pgmPoolCacheUsed(pPool, pShwPage);
1797
1798	/* Hook up the cached PD if needed (probably not given there are 512 PTs we may need sync). */
1799	if (Pdpe.u == (pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE))
1800	{ }
1801	else
1802	ASMAtomicWriteU64(&pPdpe->u, pShwPage->Core.Key \| EPT_E_READ \| EPT_E_WRITE \| EPT_E_EXECUTE);
1803	}
1804
1805	*ppPD = (PEPTPD)PGMPOOL_PAGE_2_PTR_V2(pVM, pVCpu, pShwPage);
1806	return VINF_SUCCESS;
1807	}
1808
1809
1810	#ifdef IN_RING0
1811	/**
1812	* Synchronizes a range of nested page table entries.
1813	*
1814	* The caller must own the PGM lock.
1815	*
1816	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1817	* @param GCPhys Where to start.
1818	* @param cPages How many pages which entries should be synced.
1819	* @param enmShwPagingMode The shadow paging mode (PGMMODE_EPT for VT-x,
1820	* host paging mode for AMD-V).
1821	*/
1822	int pgmShwSyncNestedPageLocked(PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint32_t cPages, PGMMODE enmShwPagingMode)
1823	{
1824	PGM_LOCK_ASSERT_OWNER(pVCpu->CTX_SUFF(pVM));
1825
1826	/** @todo r=bird: Gotta love this nested paging hacking we're still carrying with us... (Split PGM_TYPE_NESTED.) */
1827	int rc;
1828	switch (enmShwPagingMode)
1829	{
1830	case PGMMODE_32_BIT:
1831	{
1832	X86PDE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1833	rc = PGM_BTH_NAME_32BIT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1834	break;
1835	}
1836
1837	case PGMMODE_PAE:
1838	case PGMMODE_PAE_NX:
1839	{
1840	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1841	rc = PGM_BTH_NAME_PAE_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1842	break;
1843	}
1844
1845	case PGMMODE_AMD64:
1846	case PGMMODE_AMD64_NX:
1847	{
1848	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1849	rc = PGM_BTH_NAME_AMD64_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1850	break;
1851	}
1852
1853	case PGMMODE_EPT:
1854	{
1855	X86PDEPAE PdeDummy = { X86_PDE_P \| X86_PDE_US \| X86_PDE_RW \| X86_PDE_A };
1856	rc = PGM_BTH_NAME_EPT_PROT(SyncPage)(pVCpu, PdeDummy, GCPhys, cPages, ~0U /uErr/);
1857	break;
1858	}
1859
1860	default:
1861	AssertMsgFailedReturn(("%d\n", enmShwPagingMode), VERR_IPE_NOT_REACHED_DEFAULT_CASE);
1862	}
1863	return rc;
1864	}
1865	#endif /* IN_RING0 */
1866
1867
1868	/**
1869	* Gets effective Guest OS page information.
1870	*
1871	* When GCPtr is in a big page, the function will return as if it was a normal
1872	* 4KB page. If the need for distinguishing between big and normal page becomes
1873	* necessary at a later point, a PGMGstGetPage() will be created for that
1874	* purpose.
1875	*
1876	* @returns VBox status code.
1877	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1878	* @param GCPtr Guest Context virtual address of the page.
1879	* @param pfFlags Where to store the flags. These are X86_PTE_*, even for big pages.
1880	* @param pGCPhys Where to store the GC physical address of the page.
1881	* This is page aligned. The fact that the
1882	*/
1883	VMMDECL(int) PGMGstGetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, uint64_t *pfFlags, PRTGCPHYS pGCPhys)
1884	{
1885	VMCPU_ASSERT_EMT(pVCpu);
1886	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
1887	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
1888	AssertReturn(g_aPgmGuestModeData[idx].pfnGetPage, VERR_PGM_MODE_IPE);
1889	return g_aPgmGuestModeData[idx].pfnGetPage(pVCpu, GCPtr, pfFlags, pGCPhys);
1890	}
1891
1892
1893	/**
1894	* Performs a guest page table walk.
1895	*
1896	* The guest should be in paged protect mode or long mode when making a call to
1897	* this function.
1898	*
1899	* @returns VBox status code.
1900	* @retval VINF_SUCCESS on success.
1901	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1902	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1903	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1904	*
1905	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1906	* @param GCPtr The guest virtual address to walk by.
1907	* @param pWalk Where to return the walk result. This is valid for some
1908	* error codes as well.
1909	*/
1910	int pgmGstPtWalk(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1911	{
1912	VMCPU_ASSERT_EMT(pVCpu);
1913	switch (pVCpu->pgm.s.enmGuestMode)
1914	{
1915	case PGMMODE_32_BIT:
1916	pWalk->enmType = PGMPTWALKGSTTYPE_32BIT;
1917	return PGM_GST_NAME_32BIT(Walk)(pVCpu, GCPtr, &pWalk->u.Legacy);
1918
1919	case PGMMODE_PAE:
1920	case PGMMODE_PAE_NX:
1921	pWalk->enmType = PGMPTWALKGSTTYPE_PAE;
1922	return PGM_GST_NAME_PAE(Walk)(pVCpu, GCPtr, &pWalk->u.Pae);
1923
1924	case PGMMODE_AMD64:
1925	case PGMMODE_AMD64_NX:
1926	pWalk->enmType = PGMPTWALKGSTTYPE_AMD64;
1927	return PGM_GST_NAME_AMD64(Walk)(pVCpu, GCPtr, &pWalk->u.Amd64);
1928
1929	case PGMMODE_REAL:
1930	case PGMMODE_PROTECTED:
1931	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1932	return VERR_PGM_NOT_USED_IN_MODE;
1933
1934	case PGMMODE_NESTED_32BIT:
1935	case PGMMODE_NESTED_PAE:
1936	case PGMMODE_NESTED_AMD64:
1937	case PGMMODE_EPT:
1938	default:
1939	AssertFailed();
1940	pWalk->enmType = PGMPTWALKGSTTYPE_INVALID;
1941	return VERR_PGM_NOT_USED_IN_MODE;
1942	}
1943	}
1944
1945
1946	/**
1947	* Tries to continue the previous walk.
1948	*
1949	* @note Requires the caller to hold the PGM lock from the first
1950	* pgmGstPtWalk() call to the last pgmGstPtWalkNext() call. Otherwise
1951	* we cannot use the pointers.
1952	*
1953	* @returns VBox status code.
1954	* @retval VINF_SUCCESS on success.
1955	* @retval VERR_PAGE_TABLE_NOT_PRESENT on failure. Check pWalk for details.
1956	* @retval VERR_PGM_NOT_USED_IN_MODE if not paging isn't enabled. @a pWalk is
1957	* not valid, except enmType is PGMPTWALKGSTTYPE_INVALID.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
1960	* @param GCPtr The guest virtual address to walk by.
1961	* @param pWalk Pointer to the previous walk result and where to return
1962	* the result of this walk. This is valid for some error
1963	* codes as well.
1964	*/
1965	int pgmGstPtWalkNext(PVMCPUCC pVCpu, RTGCPTR GCPtr, PPGMPTWALKGST pWalk)
1966	{
1967	/*
1968	* We can only handle successfully walks.
1969	* We also limit ourselves to the next page.
1970	*/
1971	if ( pWalk->u.Core.fSucceeded
1972	&& GCPtr - pWalk->u.Core.GCPtr == PAGE_SIZE)
1973	{
1974	Assert(pWalk->u.Core.uLevel == 0);
1975	if (pWalk->enmType == PGMPTWALKGSTTYPE_AMD64)
1976	{
1977	/*
1978	* AMD64
1979	*/
1980	if (!pWalk->u.Core.fGigantPage && !pWalk->u.Core.fBigPage)
1981	{
1982	/*
1983	* We fall back to full walk if the PDE table changes, if any
1984	* reserved bits are set, or if the effective page access changes.
1985	*/
1986	const uint64_t fPteSame = X86_PTE_P \| X86_PTE_RW \| X86_PTE_US \| X86_PTE_PWT
1987	\| X86_PTE_PCD \| X86_PTE_A \| X86_PTE_PAE_NX;
1988	const uint64_t fPdeSame = X86_PDE_P \| X86_PDE_RW \| X86_PDE_US \| X86_PDE_PWT
1989	\| X86_PDE_PCD \| X86_PDE_A \| X86_PDE_PAE_NX \| X86_PDE_PS;
1990
1991	if ((GCPtr >> X86_PD_PAE_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PD_PAE_SHIFT))
1992	{
1993	if (pWalk->u.Amd64.pPte)
1994	{
1995	X86PTEPAE Pte;
1996	Pte.u = pWalk->u.Amd64.pPte[1].u;
1997	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
1998	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
1999	{
2000
2001	pWalk->u.Core.GCPtr = GCPtr;
2002	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2003	pWalk->u.Amd64.Pte.u = Pte.u;
2004	pWalk->u.Amd64.pPte++;
2005	return VINF_SUCCESS;
2006	}
2007	}
2008	}
2009	else if ((GCPtr >> X86_PDPT_SHIFT) == (pWalk->u.Core.GCPtr >> X86_PDPT_SHIFT))
2010	{
2011	Assert(!((GCPtr >> X86_PT_PAE_SHIFT) & X86_PT_PAE_MASK)); /* Must be first PT entry. */
2012	if (pWalk->u.Amd64.pPde)
2013	{
2014	X86PDEPAE Pde;
2015	Pde.u = pWalk->u.Amd64.pPde[1].u;
2016	if ( (Pde.u & fPdeSame) == (pWalk->u.Amd64.Pde.u & fPdeSame)
2017	&& !(Pde.u & (pVCpu)->pgm.s.fGstAmd64MbzPdeMask))
2018	{
2019	/* Get the new PTE and check out the first entry. */
2020	int rc = PGM_GCPHYS_2_PTR_BY_VMCPU(pVCpu, PGM_A20_APPLY(pVCpu, (Pde.u & X86_PDE_PAE_PG_MASK)),
2021	&pWalk->u.Amd64.pPt);
2022	if (RT_SUCCESS(rc))
2023	{
2024	pWalk->u.Amd64.pPte = &pWalk->u.Amd64.pPt->a[0];
2025	X86PTEPAE Pte;
2026	Pte.u = pWalk->u.Amd64.pPte->u;
2027	if ( (Pte.u & fPteSame) == (pWalk->u.Amd64.Pte.u & fPteSame)
2028	&& !(Pte.u & (pVCpu)->pgm.s.fGstAmd64MbzPteMask))
2029	{
2030	pWalk->u.Core.GCPtr = GCPtr;
2031	pWalk->u.Core.GCPhys = Pte.u & X86_PTE_PAE_PG_MASK;
2032	pWalk->u.Amd64.Pte.u = Pte.u;
2033	pWalk->u.Amd64.Pde.u = Pde.u;
2034	pWalk->u.Amd64.pPde++;
2035	return VINF_SUCCESS;
2036	}
2037	}
2038	}
2039	}
2040	}
2041	}
2042	else if (!pWalk->u.Core.fGigantPage)
2043	{
2044	if ((GCPtr & X86_PAGE_2M_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_2M_BASE_MASK))
2045	{
2046	pWalk->u.Core.GCPtr = GCPtr;
2047	pWalk->u.Core.GCPhys += PAGE_SIZE;
2048	return VINF_SUCCESS;
2049	}
2050	}
2051	else
2052	{
2053	if ((GCPtr & X86_PAGE_1G_BASE_MASK) == (pWalk->u.Core.GCPtr & X86_PAGE_1G_BASE_MASK))
2054	{
2055	pWalk->u.Core.GCPtr = GCPtr;
2056	pWalk->u.Core.GCPhys += PAGE_SIZE;
2057	return VINF_SUCCESS;
2058	}
2059	}
2060	}
2061	}
2062	/* Case we don't handle. Do full walk. */
2063	return pgmGstPtWalk(pVCpu, GCPtr, pWalk);
2064	}
2065
2066
2067	/**
2068	* Checks if the page is present.
2069	*
2070	* @returns true if the page is present.
2071	* @returns false if the page is not present.
2072	* @param pVCpu The cross context virtual CPU structure.
2073	* @param GCPtr Address within the page.
2074	*/
2075	VMMDECL(bool) PGMGstIsPagePresent(PVMCPUCC pVCpu, RTGCPTR GCPtr)
2076	{
2077	VMCPU_ASSERT_EMT(pVCpu);
2078	int rc = PGMGstGetPage(pVCpu, GCPtr, NULL, NULL);
2079	return RT_SUCCESS(rc);
2080	}
2081
2082
2083	/**
2084	* Sets (replaces) the page flags for a range of pages in the guest's tables.
2085	*
2086	* @returns VBox status code.
2087	* @param pVCpu The cross context virtual CPU structure.
2088	* @param GCPtr The address of the first page.
2089	* @param cb The size of the range in bytes.
2090	* @param fFlags Page flags X86_PTE_*, excluding the page mask of course.
2091	*/
2092	VMMDECL(int) PGMGstSetPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags)
2093	{
2094	VMCPU_ASSERT_EMT(pVCpu);
2095	return PGMGstModifyPage(pVCpu, GCPtr, cb, fFlags, 0);
2096	}
2097
2098
2099	/**
2100	* Modify page flags for a range of pages in the guest's tables
2101	*
2102	* The existing flags are ANDed with the fMask and ORed with the fFlags.
2103	*
2104	* @returns VBox status code.
2105	* @param pVCpu The cross context virtual CPU structure.
2106	* @param GCPtr Virtual address of the first page in the range.
2107	* @param cb Size (in bytes) of the range to apply the modification to.
2108	* @param fFlags The OR mask - page flags X86_PTE_*, excluding the page mask of course.
2109	* @param fMask The AND mask - page flags X86_PTE_*, excluding the page mask of course.
2110	* Be very CAREFUL when ~'ing constants which could be 32-bit!
2111	*/
2112	VMMDECL(int) PGMGstModifyPage(PVMCPUCC pVCpu, RTGCPTR GCPtr, size_t cb, uint64_t fFlags, uint64_t fMask)
2113	{
2114	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2115	VMCPU_ASSERT_EMT(pVCpu);
2116
2117	/*
2118	* Validate input.
2119	*/
2120	AssertMsg(!(fFlags & X86_PTE_PAE_PG_MASK), ("fFlags=%#llx\n", fFlags));
2121	Assert(cb);
2122
2123	LogFlow(("PGMGstModifyPage %RGv %d bytes fFlags=%08llx fMask=%08llx\n", GCPtr, cb, fFlags, fMask));
2124
2125	/*
2126	* Adjust input.
2127	*/
2128	cb += GCPtr & PAGE_OFFSET_MASK;
2129	cb = RT_ALIGN_Z(cb, PAGE_SIZE);
2130	GCPtr = (GCPtr & PAGE_BASE_GC_MASK);
2131
2132	/*
2133	* Call worker.
2134	*/
2135	uintptr_t idx = pVCpu->pgm.s.idxGuestModeData;
2136	AssertReturn(idx < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
2137	AssertReturn(g_aPgmGuestModeData[idx].pfnModifyPage, VERR_PGM_MODE_IPE);
2138	int rc = g_aPgmGuestModeData[idx].pfnModifyPage(pVCpu, GCPtr, cb, fFlags, fMask);
2139
2140	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,GstModifyPage), a);
2141	return rc;
2142	}
2143
2144
2145	/**
2146	* Checks whether the given PAE PDPEs are potentially valid for the guest.
2147	*
2148	* @returns @c true if the PDPE is valid, @c false otherwise.
2149	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2150	* @param paPaePdpes The PAE PDPEs to validate.
2151	*
2152	* @remarks This function -only- checks the reserved bits in the PDPE entries.
2153	*/
2154	VMM_INT_DECL(bool) PGMGstArePaePdpesValid(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2155	{
2156	Assert(paPaePdpes);
2157	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2158	{
2159	X86PDPE const PaePdpe = paPaePdpes[i];
2160	if ( !(PaePdpe.u & X86_PDPE_P)
2161	\|\| !(PaePdpe.u & pVCpu->pgm.s.fGstPaeMbzPdpeMask))
2162	{ /* likely */ }
2163	else
2164	return false;
2165	}
2166	return true;
2167	}
2168
2169
2170	/**
2171	* Performs the lazy mapping of the 32-bit guest PD.
2172	*
2173	* @returns VBox status code.
2174	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2175	* @param ppPd Where to return the pointer to the mapping. This is
2176	* always set.
2177	*/
2178	int pgmGstLazyMap32BitPD(PVMCPUCC pVCpu, PX86PD *ppPd)
2179	{
2180	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2181	PGM_LOCK_VOID(pVM);
2182
2183	Assert(!pVCpu->pgm.s.CTX_SUFF(pGst32BitPd));
2184
2185	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAGE_MASK;
2186	PPGMPAGE pPage;
2187	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2188	if (RT_SUCCESS(rc))
2189	{
2190	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPd);
2191	if (RT_SUCCESS(rc))
2192	{
2193	# ifdef IN_RING3
2194	pVCpu->pgm.s.pGst32BitPdR0 = NIL_RTR0PTR;
2195	pVCpu->pgm.s.pGst32BitPdR3 = *ppPd;
2196	# else
2197	pVCpu->pgm.s.pGst32BitPdR3 = NIL_RTR0PTR;
2198	pVCpu->pgm.s.pGst32BitPdR0 = *ppPd;
2199	# endif
2200	PGM_UNLOCK(pVM);
2201	return VINF_SUCCESS;
2202	}
2203	AssertRC(rc);
2204	}
2205	PGM_UNLOCK(pVM);
2206
2207	*ppPd = NULL;
2208	return rc;
2209	}
2210
2211
2212	/**
2213	* Performs the lazy mapping of the PAE guest PDPT.
2214	*
2215	* @returns VBox status code.
2216	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2217	* @param ppPdpt Where to return the pointer to the mapping. This is
2218	* always set.
2219	*/
2220	int pgmGstLazyMapPaePDPT(PVMCPUCC pVCpu, PX86PDPT *ppPdpt)
2221	{
2222	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt));
2223	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2224	PGM_LOCK_VOID(pVM);
2225
2226	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_PAE_PAGE_MASK;
2227	PPGMPAGE pPage;
2228	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2229	if (RT_SUCCESS(rc))
2230	{
2231	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPdpt);
2232	if (RT_SUCCESS(rc))
2233	{
2234	# ifdef IN_RING3
2235	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2236	pVCpu->pgm.s.pGstPaePdptR3 = *ppPdpt;
2237	# else
2238	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2239	pVCpu->pgm.s.pGstPaePdptR0 = *ppPdpt;
2240	# endif
2241	PGM_UNLOCK(pVM);
2242	return VINF_SUCCESS;
2243	}
2244	AssertRC(rc);
2245	}
2246
2247	PGM_UNLOCK(pVM);
2248	*ppPdpt = NULL;
2249	return rc;
2250	}
2251
2252
2253	/**
2254	* Performs the lazy mapping / updating of a PAE guest PD.
2255	*
2256	* @returns Pointer to the mapping.
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2259	* @param iPdpt Which PD entry to map (0..3).
2260	* @param ppPd Where to return the pointer to the mapping. This is
2261	* always set.
2262	*/
2263	int pgmGstLazyMapPaePD(PVMCPUCC pVCpu, uint32_t iPdpt, PX86PDPAE *ppPd)
2264	{
2265	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2266	PGM_LOCK_VOID(pVM);
2267
2268	PX86PDPT pGuestPDPT = pVCpu->pgm.s.CTX_SUFF(pGstPaePdpt);
2269	Assert(pGuestPDPT);
2270	Assert(pGuestPDPT->a[iPdpt].u & X86_PDPE_P);
2271	RTGCPHYS GCPhys = pGuestPDPT->a[iPdpt].u & X86_PDPE_PG_MASK;
2272	bool const fChanged = pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] != GCPhys;
2273
2274	PPGMPAGE pPage;
2275	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2276	if (RT_SUCCESS(rc))
2277	{
2278	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)ppPd);
2279	AssertRC(rc);
2280	if (RT_SUCCESS(rc))
2281	{
2282	# ifdef IN_RING3
2283	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2284	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = *ppPd;
2285	# else
2286	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2287	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = *ppPd;
2288	# endif
2289	if (fChanged)
2290	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = GCPhys;
2291	PGM_UNLOCK(pVM);
2292	return VINF_SUCCESS;
2293	}
2294	}
2295
2296	/* Invalid page or some failure, invalidate the entry. */
2297	pVCpu->pgm.s.aGCPhysGstPaePDs[iPdpt] = NIL_RTGCPHYS;
2298	pVCpu->pgm.s.apGstPaePDsR3[iPdpt] = NIL_RTR3PTR;
2299	pVCpu->pgm.s.apGstPaePDsR0[iPdpt] = NIL_RTR0PTR;
2300
2301	PGM_UNLOCK(pVM);
2302	return rc;
2303	}
2304
2305
2306	/**
2307	* Performs the lazy mapping of the 32-bit guest PD.
2308	*
2309	* @returns VBox status code.
2310	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2311	* @param ppPml4 Where to return the pointer to the mapping. This will
2312	* always be set.
2313	*/
2314	int pgmGstLazyMapPml4(PVMCPUCC pVCpu, PX86PML4 *ppPml4)
2315	{
2316	Assert(!pVCpu->pgm.s.CTX_SUFF(pGstAmd64Pml4));
2317	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2318	PGM_LOCK_VOID(pVM);
2319
2320	RTGCPHYS GCPhysCR3 = pVCpu->pgm.s.GCPhysCR3 & X86_CR3_AMD64_PAGE_MASK;
2321	PPGMPAGE pPage;
2322	int rc = pgmPhysGetPageEx(pVM, GCPhysCR3, &pPage);
2323	if (RT_SUCCESS(rc))
2324	{
2325	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhysCR3, (void **)ppPml4);
2326	if (RT_SUCCESS(rc))
2327	{
2328	# ifdef IN_RING3
2329	pVCpu->pgm.s.pGstAmd64Pml4R0 = NIL_RTR0PTR;
2330	pVCpu->pgm.s.pGstAmd64Pml4R3 = *ppPml4;
2331	# else
2332	pVCpu->pgm.s.pGstAmd64Pml4R3 = NIL_RTR3PTR;
2333	pVCpu->pgm.s.pGstAmd64Pml4R0 = *ppPml4;
2334	# endif
2335	PGM_UNLOCK(pVM);
2336	return VINF_SUCCESS;
2337	}
2338	}
2339
2340	PGM_UNLOCK(pVM);
2341	*ppPml4 = NULL;
2342	return rc;
2343	}
2344
2345
2346	/**
2347	* Gets the current CR3 register value for the shadow memory context.
2348	* @returns CR3 value.
2349	* @param pVCpu The cross context virtual CPU structure.
2350	*/
2351	VMMDECL(RTHCPHYS) PGMGetHyperCR3(PVMCPU pVCpu)
2352	{
2353	PPGMPOOLPAGE pPoolPage = pVCpu->pgm.s.CTX_SUFF(pShwPageCR3);
2354	AssertPtrReturn(pPoolPage, NIL_RTHCPHYS);
2355	return pPoolPage->Core.Key;
2356	}
2357
2358
2359	/**
2360	* Forces lazy remapping of the guest's PAE page-directory structures.
2361	*
2362	* @param pVCpu The cross context virtual CPU structure.
2363	*/
2364	static void pgmGstFlushPaePdpes(PVMCPU pVCpu)
2365	{
2366	for (unsigned i = 0; i < RT_ELEMENTS(pVCpu->pgm.s.aGCPhysGstPaePDs); i++)
2367	{
2368	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2369	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2370	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2371	}
2372	}
2373
2374
2375	/**
2376	* Gets the PGM CR3 value masked according to the current guest mode.
2377	*
2378	* @returns The masked PGM CR3 value.
2379	* @param pVCpu The cross context virtual CPU structure.
2380	* @param uCr3 The raw guest CR3 value.
2381	*/
2382	DECLINLINE(RTGCPHYS) pgmGetGuestMaskedCr3(PVMCPUCC pVCpu, uint64_t uCr3)
2383	{
2384	RTGCPHYS GCPhysCR3;
2385	switch (pVCpu->pgm.s.enmGuestMode)
2386	{
2387	case PGMMODE_PAE:
2388	case PGMMODE_PAE_NX:
2389	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_PAE_PAGE_MASK);
2390	break;
2391	case PGMMODE_AMD64:
2392	case PGMMODE_AMD64_NX:
2393	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_AMD64_PAGE_MASK);
2394	break;
2395	default:
2396	GCPhysCR3 = (RTGCPHYS)(uCr3 & X86_CR3_PAGE_MASK);
2397	break;
2398	}
2399	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2400	return GCPhysCR3;
2401	}
2402
2403
2404	/**
2405	* Performs and schedules necessary updates following a CR3 load or reload.
2406	*
2407	* This will normally involve mapping the guest PD or nPDPT
2408	*
2409	* @returns VBox status code.
2410	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync. This can
2411	* safely be ignored and overridden since the FF will be set too then.
2412	* @param pVCpu The cross context virtual CPU structure.
2413	* @param cr3 The new cr3.
2414	* @param fGlobal Indicates whether this is a global flush or not.
2415	* @param fPdpesMapped Whether the PAE PDPEs (and PDPT) have been mapped.
2416	*/
2417	VMMDECL(int) PGMFlushTLB(PVMCPUCC pVCpu, uint64_t cr3, bool fGlobal, bool fPdpesMapped)
2418	{
2419	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2420	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2421
2422	VMCPU_ASSERT_EMT(pVCpu);
2423
2424	/*
2425	* Always flag the necessary updates; necessary for hardware acceleration
2426	*/
2427	/** @todo optimize this, it shouldn't always be necessary. */
2428	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2429	if (fGlobal)
2430	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2431	LogFlow(("PGMFlushTLB: cr3=%RX64 OldCr3=%RX64 fGlobal=%d\n", cr3, pVCpu->pgm.s.GCPhysCR3, fGlobal));
2432
2433	/*
2434	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2435	*/
2436	int rc = VINF_SUCCESS;
2437	RTGCPHYS const GCPhysOldCR3 = pVCpu->pgm.s.GCPhysCR3;
2438	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2439	if (GCPhysOldCR3 != GCPhysCR3)
2440	{
2441	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2442	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2443	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2444
2445	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2446	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, fPdpesMapped);
2447	if (RT_LIKELY(rc == VINF_SUCCESS))
2448	{ }
2449	else
2450	{
2451	AssertMsg(rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc));
2452	Assert(VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_PGM_SYNC_CR3));
2453	pVCpu->pgm.s.GCPhysCR3 = GCPhysOldCR3;
2454	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_MAP_CR3;
2455	}
2456
2457	if (fGlobal)
2458	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3Global));
2459	else
2460	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBNewCR3));
2461	}
2462	else
2463	{
2464	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
2465	PPGMPOOL pPool = pVM->pgm.s.CTX_SUFF(pPool);
2466	if (pPool->cDirtyPages)
2467	{
2468	PGM_LOCK_VOID(pVM);
2469	pgmPoolResetDirtyPages(pVM);
2470	PGM_UNLOCK(pVM);
2471	}
2472	#endif
2473	if (fGlobal)
2474	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3Global));
2475	else
2476	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLBSameCR3));
2477
2478	/*
2479	* Flush PAE PDPTEs.
2480	*/
2481	if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2482	pgmGstFlushPaePdpes(pVCpu);
2483	}
2484
2485	IEMTlbInvalidateAll(pVCpu, false /fVmm/);
2486	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,FlushTLB), a);
2487	return rc;
2488	}
2489
2490
2491	/**
2492	* Performs and schedules necessary updates following a CR3 load or reload when
2493	* using nested or extended paging.
2494	*
2495	* This API is an alternative to PGMFlushTLB that avoids actually flushing the
2496	* TLB and triggering a SyncCR3.
2497	*
2498	* This will normally involve mapping the guest PD or nPDPT
2499	*
2500	* @returns VBox status code.
2501	* @retval VINF_SUCCESS.
2502	* @retval VINF_PGM_SYNC_CR3 if monitoring requires a CR3 sync (not for nested
2503	* paging modes). This can safely be ignored and overridden since the
2504	* FF will be set too then.
2505	* @param pVCpu The cross context virtual CPU structure.
2506	* @param cr3 The new CR3.
2507	* @param fPdpesMapped Whether the PAE PDPEs (and PDPT) have been mapped.
2508	*/
2509	VMMDECL(int) PGMUpdateCR3(PVMCPUCC pVCpu, uint64_t cr3, bool fPdpesMapped)
2510	{
2511	VMCPU_ASSERT_EMT(pVCpu);
2512	LogFlow(("PGMUpdateCR3: cr3=%RX64 OldCr3=%RX64\n", cr3, pVCpu->pgm.s.GCPhysCR3));
2513
2514	/* We assume we're only called in nested paging mode. */
2515	Assert(pVCpu->CTX_SUFF(pVM)->pgm.s.fNestedPaging \|\| pVCpu->pgm.s.enmShadowMode == PGMMODE_EPT);
2516
2517	/*
2518	* Remap the CR3 content and adjust the monitoring if CR3 was actually changed.
2519	*/
2520	int rc = VINF_SUCCESS;
2521	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2522	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2523	{
2524	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2525	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2526	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2527
2528	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2529	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, fPdpesMapped);
2530
2531	AssertRCSuccess(rc); /* Assumes VINF_PGM_SYNC_CR3 doesn't apply to nested paging. / /* @todo this isn't true for the mac, but we need hw to test/fix this. */
2532	}
2533	/*
2534	* Flush PAE PDPTEs.
2535	*/
2536	else if (PGMMODE_IS_PAE(pVCpu->pgm.s.enmGuestMode))
2537	pgmGstFlushPaePdpes(pVCpu);
2538
2539	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
2540	return rc;
2541	}
2542
2543
2544	/**
2545	* Synchronize the paging structures.
2546	*
2547	* This function is called in response to the VM_FF_PGM_SYNC_CR3 and
2548	* VM_FF_PGM_SYNC_CR3_NONGLOBAL. Those two force action flags are set
2549	* in several places, most importantly whenever the CR3 is loaded.
2550	*
2551	* @returns VBox status code. May return VINF_PGM_SYNC_CR3 in RC/R0.
2552	* @retval VERR_PGM_NO_HYPERVISOR_ADDRESS in raw-mode when we're unable to map
2553	* the VMM into guest context.
2554	* @param pVCpu The cross context virtual CPU structure.
2555	* @param cr0 Guest context CR0 register
2556	* @param cr3 Guest context CR3 register
2557	* @param cr4 Guest context CR4 register
2558	* @param fGlobal Including global page directories or not
2559	*/
2560	VMMDECL(int) PGMSyncCR3(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr3, uint64_t cr4, bool fGlobal)
2561	{
2562	int rc;
2563
2564	VMCPU_ASSERT_EMT(pVCpu);
2565
2566	/*
2567	* The pool may have pending stuff and even require a return to ring-3 to
2568	* clear the whole thing.
2569	*/
2570	rc = pgmPoolSyncCR3(pVCpu);
2571	if (rc != VINF_SUCCESS)
2572	return rc;
2573
2574	/*
2575	* We might be called when we shouldn't.
2576	*
2577	* The mode switching will ensure that the PD is resynced after every mode
2578	* switch. So, if we find ourselves here when in protected or real mode
2579	* we can safely clear the FF and return immediately.
2580	*/
2581	if (pVCpu->pgm.s.enmGuestMode <= PGMMODE_PROTECTED)
2582	{
2583	Assert((cr0 & (X86_CR0_PG \| X86_CR0_PE)) != (X86_CR0_PG \| X86_CR0_PE));
2584	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2585	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2586	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2587	return VINF_SUCCESS;
2588	}
2589
2590	/* If global pages are not supported, then all flushes are global. */
2591	if (!(cr4 & X86_CR4_PGE))
2592	fGlobal = true;
2593	LogFlow(("PGMSyncCR3: cr0=%RX64 cr3=%RX64 cr4=%RX64 fGlobal=%d[%d,%d]\n", cr0, cr3, cr4, fGlobal,
2594	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3), VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL)));
2595
2596	/*
2597	* Check if we need to finish an aborted MapCR3 call (see PGMFlushTLB).
2598	* This should be done before SyncCR3.
2599	*/
2600	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_MAP_CR3)
2601	{
2602	pVCpu->pgm.s.fSyncFlags &= ~PGM_SYNC_MAP_CR3;
2603
2604	RTGCPHYS const GCPhysCR3Old = pVCpu->pgm.s.GCPhysCR3; NOREF(GCPhysCR3Old);
2605	RTGCPHYS const GCPhysCR3 = pgmGetGuestMaskedCr3(pVCpu, cr3);
2606	if (pVCpu->pgm.s.GCPhysCR3 != GCPhysCR3)
2607	{
2608	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2609	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2610	AssertReturn(g_aPgmBothModeData[idxBth].pfnMapCR3, VERR_PGM_MODE_IPE);
2611	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
2612	rc = g_aPgmBothModeData[idxBth].pfnMapCR3(pVCpu, GCPhysCR3, false /* fPdpesMapped */);
2613	}
2614
2615	/* Make sure we check for pending pgm pool syncs as we clear VMCPU_FF_PGM_SYNC_CR3 later on! */
2616	if ( rc == VINF_PGM_SYNC_CR3
2617	\|\| (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL))
2618	{
2619	Log(("PGMSyncCR3: pending pgm pool sync after MapCR3!\n"));
2620	#ifdef IN_RING3
2621	rc = pgmPoolSyncCR3(pVCpu);
2622	#else
2623	if (rc == VINF_PGM_SYNC_CR3)
2624	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3Old;
2625	return VINF_PGM_SYNC_CR3;
2626	#endif
2627	}
2628	AssertRCReturn(rc, rc);
2629	AssertRCSuccessReturn(rc, VERR_IPE_UNEXPECTED_INFO_STATUS);
2630	}
2631
2632	/*
2633	* Let the 'Bth' function do the work and we'll just keep track of the flags.
2634	*/
2635	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2636
2637	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
2638	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), VERR_PGM_MODE_IPE);
2639	AssertReturn(g_aPgmBothModeData[idxBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
2640	rc = g_aPgmBothModeData[idxBth].pfnSyncCR3(pVCpu, cr0, cr3, cr4, fGlobal);
2641
2642	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
2643	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 \|\| RT_FAILURE(rc), ("rc=%Rrc\n", rc));
2644	if (rc == VINF_SUCCESS)
2645	{
2646	if (pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL)
2647	{
2648	/* Go back to ring 3 if a pgm pool sync is again pending. */
2649	return VINF_PGM_SYNC_CR3;
2650	}
2651
2652	if (!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_ALWAYS))
2653	{
2654	Assert(!(pVCpu->pgm.s.fSyncFlags & PGM_SYNC_CLEAR_PGM_POOL));
2655	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2656	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
2657	}
2658	}
2659
2660	/*
2661	* Now flush the CR3 (guest context).
2662	*/
2663	if (rc == VINF_SUCCESS)
2664	PGM_INVL_VCPU_TLBS(pVCpu);
2665	return rc;
2666	}
2667
2668
2669	/**
2670	* Maps all the PAE PDPE entries.
2671	*
2672	* @returns VBox status code.
2673	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2674	* @param paPaePdpes The new PAE PDPE values.
2675	*
2676	* @remarks This function may be invoked during the process of changing the guest
2677	* paging mode to PAE, hence the guest state (CR0, CR4 etc.) may not
2678	* reflect PAE paging just yet.
2679	*/
2680	VMM_INT_DECL(int) PGMGstMapPaePdpes(PVMCPUCC pVCpu, PCX86PDPE paPaePdpes)
2681	{
2682	Assert(paPaePdpes);
2683	for (unsigned i = 0; i < X86_PG_PAE_PDPE_ENTRIES; i++)
2684	{
2685	X86PDPE const PaePdpe = paPaePdpes[i];
2686
2687	/*
2688	* In some cases (e.g. in SVM with nested paging) the validation of the PAE PDPEs
2689	* are deferred.[1] Also, different situations require different handling of invalid
2690	* PDPE entries. Here we assume the caller has already validated or doesn't require
2691	* validation of the PDPEs.
2692	*
2693	* [1] -- See AMD spec. 15.25.10 "Legacy PAE Mode".
2694	*/
2695	if ((PaePdpe.u & (pVCpu->pgm.s.fGstPaeMbzPdpeMask \| X86_PDPE_P)) == X86_PDPE_P)
2696	{
2697	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2698	RTHCPTR HCPtr;
2699	RTGCPHYS const GCPhys = PGM_A20_APPLY(pVCpu, PaePdpe.u & X86_PDPE_PG_MASK);
2700
2701	PGM_LOCK_VOID(pVM);
2702	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2703	AssertReturnStmt(pPage, PGM_UNLOCK(pVM), VERR_PGM_INVALID_PDPE_ADDR);
2704	int const rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)&HCPtr);
2705	PGM_UNLOCK(pVM);
2706	if (RT_SUCCESS(rc))
2707	{
2708	# ifdef IN_RING3
2709	pVCpu->pgm.s.apGstPaePDsR3[i] = (PX86PDPAE)HCPtr;
2710	pVCpu->pgm.s.apGstPaePDsR0[i] = NIL_RTR0PTR;
2711	# else
2712	pVCpu->pgm.s.apGstPaePDsR3[i] = NIL_RTR3PTR;
2713	pVCpu->pgm.s.apGstPaePDsR0[i] = (PX86PDPAE)HCPtr;
2714	# endif
2715	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = GCPhys;
2716	continue;
2717	}
2718	AssertMsgFailed(("PGMPhysMapPaePdpes: rc2=%d GCPhys=%RGp i=%d\n", rc, GCPhys, i));
2719	}
2720	pVCpu->pgm.s.apGstPaePDsR3[i] = 0;
2721	pVCpu->pgm.s.apGstPaePDsR0[i] = 0;
2722	pVCpu->pgm.s.aGCPhysGstPaePDs[i] = NIL_RTGCPHYS;
2723	}
2724
2725	return VINF_SUCCESS;
2726	}
2727
2728
2729	/**
2730	* Validates and maps the PDPT and PAE PDPEs referenced by the given CR3.
2731	*
2732	* @returns VBox status code.
2733	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2734	* @param cr3 The guest CR3 value.
2735	*
2736	* @remarks This function may be invoked during the process of changing the guest
2737	* paging mode to PAE but the guest state (CR0, CR4 etc.) may not reflect
2738	* PAE paging just yet.
2739	*/
2740	VMM_INT_DECL(int) PGMGstMapPaePdpesAtCr3(PVMCPUCC pVCpu, uint64_t cr3)
2741	{
2742	/*
2743	* Read the page-directory-pointer table (PDPT) at CR3.
2744	*/
2745	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
2746	RTGCPHYS GCPhysCR3 = (cr3 & X86_CR3_PAE_PAGE_MASK);
2747	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhysCR3);
2748
2749	PGM_LOCK_VOID(pVM);
2750	PPGMPAGE pPageCR3 = pgmPhysGetPage(pVM, GCPhysCR3);
2751	AssertReturnStmt(pPageCR3, PGM_UNLOCK(pVM), VERR_PGM_INVALID_CR3_ADDR);
2752
2753	X86PDPE aPaePdpes[X86_PG_PAE_PDPE_ENTRIES];
2754	RTHCPTR HCPtrGuestCr3;
2755	int rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPageCR3, GCPhysCR3, (void **)&HCPtrGuestCr3);
2756	PGM_UNLOCK(pVM);
2757	AssertRCReturn(rc, rc);
2758	memcpy(&aPaePdpes[0], HCPtrGuestCr3, sizeof(aPaePdpes));
2759
2760	/*
2761	* Validate the page-directory-pointer table entries (PDPE).
2762	*/
2763	if (PGMGstArePaePdpesValid(pVCpu, &aPaePdpes[0]))
2764	{
2765	/*
2766	* Map the PDPT.
2767	* We deliberately don't update PGM's GCPhysCR3 here as it's expected
2768	* that PGMFlushTLB will be called soon and only a change to CR3 then
2769	* will cause the shadow page tables to be updated.
2770	*/
2771	# ifdef IN_RING3
2772	pVCpu->pgm.s.pGstPaePdptR3 = (PX86PDPT)HCPtrGuestCr3;
2773	pVCpu->pgm.s.pGstPaePdptR0 = NIL_RTR0PTR;
2774	# else
2775	pVCpu->pgm.s.pGstPaePdptR3 = NIL_RTR3PTR;
2776	pVCpu->pgm.s.pGstPaePdptR0 = (PX86PDPT)HCPtrGuestCr3;
2777	# endif
2778
2779	/*
2780	* Update CPUM.
2781	* We do this prior to mapping the PDPEs to keep the order consistent
2782	* with what's used in HM. In practice, it doesn't really matter.
2783	*/
2784	CPUMSetGuestPaePdpes(pVCpu, &aPaePdpes[0]);
2785
2786	/*
2787	* Map the PDPEs.
2788	*/
2789	return PGMGstMapPaePdpes(pVCpu, &aPaePdpes[0]);
2790	}
2791	return VERR_PGM_PAE_PDPE_RSVD;
2792	}
2793
2794
2795	/**
2796	* Called whenever CR0 or CR4 in a way which may affect the paging mode.
2797	*
2798	* @returns VBox status code, with the following informational code for
2799	* VM scheduling.
2800	* @retval VINF_SUCCESS if the was no change, or it was successfully dealt with.
2801	* @retval VINF_PGM_CHANGE_MODE if we're in RC the mode changes. This will
2802	* NOT be returned in ring-3 or ring-0.
2803	* @retval VINF_EM_SUSPEND or VINF_EM_OFF on a fatal runtime error. (R3 only)
2804	*
2805	* @param pVCpu The cross context virtual CPU structure.
2806	* @param cr0 The new cr0.
2807	* @param cr4 The new cr4.
2808	* @param efer The new extended feature enable register.
2809	*/
2810	VMMDECL(int) PGMChangeMode(PVMCPUCC pVCpu, uint64_t cr0, uint64_t cr4, uint64_t efer)
2811	{
2812	VMCPU_ASSERT_EMT(pVCpu);
2813
2814	/*
2815	* Calc the new guest mode.
2816	*
2817	* Note! We check PG before PE and without requiring PE because of the
2818	* special AMD-V paged real mode (APM vol 2, rev 3.28, 15.9).
2819	*/
2820	PGMMODE enmGuestMode;
2821	if (cr0 & X86_CR0_PG)
2822	{
2823	if (!(cr4 & X86_CR4_PAE))
2824	{
2825	bool const fPse = !!(cr4 & X86_CR4_PSE);
2826	if (pVCpu->pgm.s.fGst32BitPageSizeExtension != fPse)
2827	Log(("PGMChangeMode: CR4.PSE %d -> %d\n", pVCpu->pgm.s.fGst32BitPageSizeExtension, fPse));
2828	pVCpu->pgm.s.fGst32BitPageSizeExtension = fPse;
2829	enmGuestMode = PGMMODE_32_BIT;
2830	}
2831	else if (!(efer & MSR_K6_EFER_LME))
2832	{
2833	if (!(efer & MSR_K6_EFER_NXE))
2834	enmGuestMode = PGMMODE_PAE;
2835	else
2836	enmGuestMode = PGMMODE_PAE_NX;
2837	}
2838	else
2839	{
2840	if (!(efer & MSR_K6_EFER_NXE))
2841	enmGuestMode = PGMMODE_AMD64;
2842	else
2843	enmGuestMode = PGMMODE_AMD64_NX;
2844	}
2845	}
2846	else if (!(cr0 & X86_CR0_PE))
2847	enmGuestMode = PGMMODE_REAL;
2848	else
2849	enmGuestMode = PGMMODE_PROTECTED;
2850
2851	/*
2852	* Did it change?
2853	*/
2854	if (pVCpu->pgm.s.enmGuestMode == enmGuestMode)
2855	return VINF_SUCCESS;
2856
2857	/* Flush the TLB */
2858	PGM_INVL_VCPU_TLBS(pVCpu);
2859	return PGMHCChangeMode(pVCpu->CTX_SUFF(pVM), pVCpu, enmGuestMode);
2860	}
2861
2862
2863	/**
2864	* Converts a PGMMODE value to a PGM_TYPE_* \#define.
2865	*
2866	* @returns PGM_TYPE_*.
2867	* @param pgmMode The mode value to convert.
2868	*/
2869	DECLINLINE(unsigned) pgmModeToType(PGMMODE pgmMode)
2870	{
2871	switch (pgmMode)
2872	{
2873	case PGMMODE_REAL: return PGM_TYPE_REAL;
2874	case PGMMODE_PROTECTED: return PGM_TYPE_PROT;
2875	case PGMMODE_32_BIT: return PGM_TYPE_32BIT;
2876	case PGMMODE_PAE:
2877	case PGMMODE_PAE_NX: return PGM_TYPE_PAE;
2878	case PGMMODE_AMD64:
2879	case PGMMODE_AMD64_NX: return PGM_TYPE_AMD64;
2880	case PGMMODE_NESTED_32BIT: return PGM_TYPE_NESTED_32BIT;
2881	case PGMMODE_NESTED_PAE: return PGM_TYPE_NESTED_PAE;
2882	case PGMMODE_NESTED_AMD64: return PGM_TYPE_NESTED_AMD64;
2883	case PGMMODE_EPT: return PGM_TYPE_EPT;
2884	case PGMMODE_NONE: return PGM_TYPE_NONE;
2885	default:
2886	AssertFatalMsgFailed(("pgmMode=%d\n", pgmMode));
2887	}
2888	}
2889
2890
2891	/**
2892	* Calculates the shadow paging mode.
2893	*
2894	* @returns The shadow paging mode.
2895	* @param pVM The cross context VM structure.
2896	* @param enmGuestMode The guest mode.
2897	* @param enmHostMode The host mode.
2898	* @param enmShadowMode The current shadow mode.
2899	*/
2900	static PGMMODE pgmCalcShadowMode(PVMCC pVM, PGMMODE enmGuestMode, SUPPAGINGMODE enmHostMode, PGMMODE enmShadowMode)
2901	{
2902	switch (enmGuestMode)
2903	{
2904	/*
2905	* When switching to real or protected mode we don't change
2906	* anything since it's likely that we'll switch back pretty soon.
2907	*
2908	* During pgmR3InitPaging we'll end up here with PGMMODE_INVALID
2909	* and is supposed to determine which shadow paging and switcher to
2910	* use during init.
2911	*/
2912	case PGMMODE_REAL:
2913	case PGMMODE_PROTECTED:
2914	if ( enmShadowMode != PGMMODE_INVALID
2915	&& VM_IS_RAW_MODE_ENABLED(pVM) /* always switch in hm and nem modes! */)
2916	break; /* (no change) */
2917
2918	switch (enmHostMode)
2919	{
2920	case SUPPAGINGMODE_32_BIT:
2921	case SUPPAGINGMODE_32_BIT_GLOBAL:
2922	enmShadowMode = PGMMODE_32_BIT;
2923	break;
2924
2925	case SUPPAGINGMODE_PAE:
2926	case SUPPAGINGMODE_PAE_NX:
2927	case SUPPAGINGMODE_PAE_GLOBAL:
2928	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2929	enmShadowMode = PGMMODE_PAE;
2930	break;
2931
2932	case SUPPAGINGMODE_AMD64:
2933	case SUPPAGINGMODE_AMD64_GLOBAL:
2934	case SUPPAGINGMODE_AMD64_NX:
2935	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2936	enmShadowMode = PGMMODE_PAE;
2937	break;
2938
2939	default:
2940	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2941	}
2942	break;
2943
2944	case PGMMODE_32_BIT:
2945	switch (enmHostMode)
2946	{
2947	case SUPPAGINGMODE_32_BIT:
2948	case SUPPAGINGMODE_32_BIT_GLOBAL:
2949	enmShadowMode = PGMMODE_32_BIT;
2950	break;
2951
2952	case SUPPAGINGMODE_PAE:
2953	case SUPPAGINGMODE_PAE_NX:
2954	case SUPPAGINGMODE_PAE_GLOBAL:
2955	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2956	enmShadowMode = PGMMODE_PAE;
2957	break;
2958
2959	case SUPPAGINGMODE_AMD64:
2960	case SUPPAGINGMODE_AMD64_GLOBAL:
2961	case SUPPAGINGMODE_AMD64_NX:
2962	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2963	enmShadowMode = PGMMODE_PAE;
2964	break;
2965
2966	default:
2967	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2968	}
2969	break;
2970
2971	case PGMMODE_PAE:
2972	case PGMMODE_PAE_NX: /** @todo This might require more switchers and guest+both modes. */
2973	switch (enmHostMode)
2974	{
2975	case SUPPAGINGMODE_32_BIT:
2976	case SUPPAGINGMODE_32_BIT_GLOBAL:
2977	enmShadowMode = PGMMODE_PAE;
2978	break;
2979
2980	case SUPPAGINGMODE_PAE:
2981	case SUPPAGINGMODE_PAE_NX:
2982	case SUPPAGINGMODE_PAE_GLOBAL:
2983	case SUPPAGINGMODE_PAE_GLOBAL_NX:
2984	enmShadowMode = PGMMODE_PAE;
2985	break;
2986
2987	case SUPPAGINGMODE_AMD64:
2988	case SUPPAGINGMODE_AMD64_GLOBAL:
2989	case SUPPAGINGMODE_AMD64_NX:
2990	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
2991	enmShadowMode = PGMMODE_PAE;
2992	break;
2993
2994	default:
2995	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
2996	}
2997	break;
2998
2999	case PGMMODE_AMD64:
3000	case PGMMODE_AMD64_NX:
3001	switch (enmHostMode)
3002	{
3003	case SUPPAGINGMODE_32_BIT:
3004	case SUPPAGINGMODE_32_BIT_GLOBAL:
3005	enmShadowMode = PGMMODE_AMD64;
3006	break;
3007
3008	case SUPPAGINGMODE_PAE:
3009	case SUPPAGINGMODE_PAE_NX:
3010	case SUPPAGINGMODE_PAE_GLOBAL:
3011	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3012	enmShadowMode = PGMMODE_AMD64;
3013	break;
3014
3015	case SUPPAGINGMODE_AMD64:
3016	case SUPPAGINGMODE_AMD64_GLOBAL:
3017	case SUPPAGINGMODE_AMD64_NX:
3018	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3019	enmShadowMode = PGMMODE_AMD64;
3020	break;
3021
3022	default:
3023	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", enmHostMode), PGMMODE_INVALID);
3024	}
3025	break;
3026
3027	default:
3028	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), PGMMODE_INVALID);
3029	}
3030
3031	/*
3032	* Override the shadow mode when NEM or nested paging is active.
3033	*/
3034	if (VM_IS_NEM_ENABLED(pVM))
3035	{
3036	pVM->pgm.s.fNestedPaging = true;
3037	enmShadowMode = PGMMODE_NONE;
3038	}
3039	else
3040	{
3041	bool fNestedPaging = HMIsNestedPagingActive(pVM);
3042	pVM->pgm.s.fNestedPaging = fNestedPaging;
3043	if (fNestedPaging)
3044	{
3045	if (HMIsVmxActive(pVM))
3046	enmShadowMode = PGMMODE_EPT;
3047	else
3048	{
3049	/* The nested SVM paging depends on the host one. */
3050	Assert(HMIsSvmActive(pVM));
3051	if ( enmGuestMode == PGMMODE_AMD64
3052	\|\| enmGuestMode == PGMMODE_AMD64_NX)
3053	enmShadowMode = PGMMODE_NESTED_AMD64;
3054	else
3055	switch (pVM->pgm.s.enmHostMode)
3056	{
3057	case SUPPAGINGMODE_32_BIT:
3058	case SUPPAGINGMODE_32_BIT_GLOBAL:
3059	enmShadowMode = PGMMODE_NESTED_32BIT;
3060	break;
3061
3062	case SUPPAGINGMODE_PAE:
3063	case SUPPAGINGMODE_PAE_GLOBAL:
3064	case SUPPAGINGMODE_PAE_NX:
3065	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3066	enmShadowMode = PGMMODE_NESTED_PAE;
3067	break;
3068
3069	case SUPPAGINGMODE_AMD64:
3070	case SUPPAGINGMODE_AMD64_GLOBAL:
3071	case SUPPAGINGMODE_AMD64_NX:
3072	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3073	enmShadowMode = PGMMODE_NESTED_AMD64;
3074	break;
3075
3076	default:
3077	AssertLogRelMsgFailedReturn(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode), PGMMODE_INVALID);
3078	}
3079	}
3080	}
3081	}
3082
3083	return enmShadowMode;
3084	}
3085
3086
3087	/**
3088	* Performs the actual mode change.
3089	* This is called by PGMChangeMode and pgmR3InitPaging().
3090	*
3091	* @returns VBox status code. May suspend or power off the VM on error, but this
3092	* will trigger using FFs and not informational status codes.
3093	*
3094	* @param pVM The cross context VM structure.
3095	* @param pVCpu The cross context virtual CPU structure.
3096	* @param enmGuestMode The new guest mode. This is assumed to be different from
3097	* the current mode.
3098	*/
3099	VMM_INT_DECL(int) PGMHCChangeMode(PVMCC pVM, PVMCPUCC pVCpu, PGMMODE enmGuestMode)
3100	{
3101	Log(("PGMHCChangeMode: Guest mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmGuestMode), PGMGetModeName(enmGuestMode)));
3102	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cGuestModeChanges);
3103
3104	/*
3105	* Calc the shadow mode and switcher.
3106	*/
3107	PGMMODE enmShadowMode = pgmCalcShadowMode(pVM, enmGuestMode, pVM->pgm.s.enmHostMode, pVCpu->pgm.s.enmShadowMode);
3108
3109	/*
3110	* Exit old mode(s).
3111	*/
3112	/* shadow */
3113	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3114	{
3115	LogFlow(("PGMHCChangeMode: Shadow mode: %s -> %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode), PGMGetModeName(enmShadowMode)));
3116	uintptr_t idxOldShw = pVCpu->pgm.s.idxShadowModeData;
3117	if ( idxOldShw < RT_ELEMENTS(g_aPgmShadowModeData)
3118	&& g_aPgmShadowModeData[idxOldShw].pfnExit)
3119	{
3120	int rc = g_aPgmShadowModeData[idxOldShw].pfnExit(pVCpu);
3121	AssertMsgRCReturn(rc, ("Exit failed for shadow mode %d: %Rrc\n", pVCpu->pgm.s.enmShadowMode, rc), rc);
3122	}
3123	}
3124	else
3125	LogFlow(("PGMHCChangeMode: Shadow mode remains: %s\n", PGMGetModeName(pVCpu->pgm.s.enmShadowMode)));
3126
3127	/* guest */
3128	uintptr_t const idxOldGst = pVCpu->pgm.s.idxGuestModeData;
3129	if ( idxOldGst < RT_ELEMENTS(g_aPgmGuestModeData)
3130	&& g_aPgmGuestModeData[idxOldGst].pfnExit)
3131	{
3132	int rc = g_aPgmGuestModeData[idxOldGst].pfnExit(pVCpu);
3133	AssertMsgReturn(RT_SUCCESS(rc), ("Exit failed for guest mode %d: %Rrc\n", pVCpu->pgm.s.enmGuestMode, rc), rc);
3134	}
3135	pVCpu->pgm.s.GCPhysCR3 = NIL_RTGCPHYS;
3136
3137	/*
3138	* Change the paging mode data indexes.
3139	*/
3140	uintptr_t idxNewGst = pVCpu->pgm.s.idxGuestModeData = pgmModeToType(enmGuestMode);
3141	AssertReturn(idxNewGst < RT_ELEMENTS(g_aPgmGuestModeData), VERR_PGM_MODE_IPE);
3142	AssertReturn(g_aPgmGuestModeData[idxNewGst].uType == idxNewGst, VERR_PGM_MODE_IPE);
3143	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnGetPage, VERR_PGM_MODE_IPE);
3144	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnModifyPage, VERR_PGM_MODE_IPE);
3145	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnExit, VERR_PGM_MODE_IPE);
3146	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnEnter, VERR_PGM_MODE_IPE);
3147	#ifdef IN_RING3
3148	AssertPtrReturn(g_aPgmGuestModeData[idxNewGst].pfnRelocate, VERR_PGM_MODE_IPE);
3149	#endif
3150
3151	uintptr_t const idxNewShw = pVCpu->pgm.s.idxShadowModeData = pgmModeToType(enmShadowMode);
3152	AssertReturn(idxNewShw < RT_ELEMENTS(g_aPgmShadowModeData), VERR_PGM_MODE_IPE);
3153	AssertReturn(g_aPgmShadowModeData[idxNewShw].uType == idxNewShw, VERR_PGM_MODE_IPE);
3154	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnGetPage, VERR_PGM_MODE_IPE);
3155	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnModifyPage, VERR_PGM_MODE_IPE);
3156	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnExit, VERR_PGM_MODE_IPE);
3157	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnEnter, VERR_PGM_MODE_IPE);
3158	#ifdef IN_RING3
3159	AssertPtrReturn(g_aPgmShadowModeData[idxNewShw].pfnRelocate, VERR_PGM_MODE_IPE);
3160	#endif
3161
3162	uintptr_t const idxNewBth = pVCpu->pgm.s.idxBothModeData = (idxNewShw - PGM_TYPE_FIRST_SHADOW) * PGM_TYPE_END + idxNewGst;
3163	AssertReturn(g_aPgmBothModeData[idxNewBth].uShwType == idxNewShw, VERR_PGM_MODE_IPE);
3164	AssertReturn(g_aPgmBothModeData[idxNewBth].uGstType == idxNewGst, VERR_PGM_MODE_IPE);
3165	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnInvalidatePage, VERR_PGM_MODE_IPE);
3166	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnSyncCR3, VERR_PGM_MODE_IPE);
3167	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnPrefetchPage, VERR_PGM_MODE_IPE);
3168	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnVerifyAccessSyncPage, VERR_PGM_MODE_IPE);
3169	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnMapCR3, VERR_PGM_MODE_IPE);
3170	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnUnmapCR3, VERR_PGM_MODE_IPE);
3171	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnEnter, VERR_PGM_MODE_IPE);
3172	#ifdef VBOX_STRICT
3173	AssertPtrReturn(g_aPgmBothModeData[idxNewBth].pfnAssertCR3, VERR_PGM_MODE_IPE);
3174	#endif
3175
3176	/*
3177	* Enter new shadow mode (if changed).
3178	*/
3179	if (enmShadowMode != pVCpu->pgm.s.enmShadowMode)
3180	{
3181	pVCpu->pgm.s.enmShadowMode = enmShadowMode;
3182	int rc = g_aPgmShadowModeData[idxNewShw].pfnEnter(pVCpu, enmGuestMode >= PGMMODE_AMD64);
3183	AssertLogRelMsgRCReturnStmt(rc, ("Entering enmShadowMode=%s failed: %Rrc\n", PGMGetModeName(enmShadowMode), rc),
3184	pVCpu->pgm.s.enmShadowMode = PGMMODE_INVALID, rc);
3185	}
3186
3187	/*
3188	* Always flag the necessary updates
3189	*/
3190	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3191
3192	/*
3193	* Enter the new guest and shadow+guest modes.
3194	*/
3195	/* Calc the new CR3 value. */
3196	RTGCPHYS GCPhysCR3;
3197	switch (enmGuestMode)
3198	{
3199	case PGMMODE_REAL:
3200	case PGMMODE_PROTECTED:
3201	GCPhysCR3 = NIL_RTGCPHYS;
3202	break;
3203
3204	case PGMMODE_32_BIT:
3205	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAGE_MASK;
3206	break;
3207
3208	case PGMMODE_PAE_NX:
3209	case PGMMODE_PAE:
3210	if (!pVM->cpum.ro.GuestFeatures.fPae)
3211	#ifdef IN_RING3 /** @todo r=bird: wrong place, probably hasn't really worked for a while. */
3212	return VMSetRuntimeError(pVM, VMSETRTERR_FLAGS_FATAL, "PAEmode",
3213	N_("The guest is trying to switch to the PAE mode which is currently disabled by default in VirtualBox. PAE support can be enabled using the VM settings (System/Processor)"));
3214	#else
3215	AssertLogRelMsgFailedReturn(("enmGuestMode=%s - Try enable PAE for the guest!\n", PGMGetModeName(enmGuestMode)), VERR_PGM_MODE_IPE);
3216
3217	#endif
3218	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_PAE_PAGE_MASK;
3219	break;
3220
3221	#ifdef VBOX_WITH_64_BITS_GUESTS
3222	case PGMMODE_AMD64_NX:
3223	case PGMMODE_AMD64:
3224	GCPhysCR3 = CPUMGetGuestCR3(pVCpu) & X86_CR3_AMD64_PAGE_MASK;
3225	break;
3226	#endif
3227	default:
3228	AssertLogRelMsgFailedReturn(("enmGuestMode=%d\n", enmGuestMode), VERR_PGM_MODE_IPE);
3229	}
3230
3231	/* Enter the new guest mode. */
3232	pVCpu->pgm.s.enmGuestMode = enmGuestMode;
3233	int rc = g_aPgmGuestModeData[idxNewGst].pfnEnter(pVCpu, GCPhysCR3);
3234	int rc2 = g_aPgmBothModeData[idxNewBth].pfnEnter(pVCpu, GCPhysCR3);
3235
3236	/* Set the new guest CR3. */
3237	pVCpu->pgm.s.GCPhysCR3 = GCPhysCR3;
3238
3239	/* status codes. */
3240	AssertRC(rc);
3241	AssertRC(rc2);
3242	if (RT_SUCCESS(rc))
3243	{
3244	rc = rc2;
3245	if (RT_SUCCESS(rc)) /* no informational status codes. */
3246	rc = VINF_SUCCESS;
3247	}
3248
3249	/*
3250	* Notify HM.
3251	*/
3252	HMHCChangedPagingMode(pVM, pVCpu, pVCpu->pgm.s.enmShadowMode, pVCpu->pgm.s.enmGuestMode);
3253	return rc;
3254	}
3255
3256
3257	/**
3258	* Called by CPUM or REM when CR0.WP changes to 1.
3259	*
3260	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
3261	* @thread EMT
3262	*/
3263	VMMDECL(void) PGMCr0WpEnabled(PVMCPUCC pVCpu)
3264	{
3265	/*
3266	* Netware WP0+RO+US hack cleanup when WP0 -> WP1.
3267	*
3268	* Use the counter to judge whether there might be pool pages with active
3269	* hacks in them. If there are, we will be running the risk of messing up
3270	* the guest by allowing it to write to read-only pages. Thus, we have to
3271	* clear the page pool ASAP if there is the slightest chance.
3272	*/
3273	if (pVCpu->pgm.s.cNetwareWp0Hacks > 0)
3274	{
3275	Assert(pVCpu->CTX_SUFF(pVM)->cCpus == 1);
3276
3277	Log(("PGMCr0WpEnabled: %llu WP0 hacks active - clearing page pool\n", pVCpu->pgm.s.cNetwareWp0Hacks));
3278	pVCpu->pgm.s.cNetwareWp0Hacks = 0;
3279	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3280	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3281	}
3282	}
3283
3284
3285	/**
3286	* Gets the current guest paging mode.
3287	*
3288	* If you just need the CPU mode (real/protected/long), use CPUMGetGuestMode().
3289	*
3290	* @returns The current paging mode.
3291	* @param pVCpu The cross context virtual CPU structure.
3292	*/
3293	VMMDECL(PGMMODE) PGMGetGuestMode(PVMCPU pVCpu)
3294	{
3295	return pVCpu->pgm.s.enmGuestMode;
3296	}
3297
3298
3299	/**
3300	* Gets the current shadow paging mode.
3301	*
3302	* @returns The current paging mode.
3303	* @param pVCpu The cross context virtual CPU structure.
3304	*/
3305	VMMDECL(PGMMODE) PGMGetShadowMode(PVMCPU pVCpu)
3306	{
3307	return pVCpu->pgm.s.enmShadowMode;
3308	}
3309
3310
3311	/**
3312	* Gets the current host paging mode.
3313	*
3314	* @returns The current paging mode.
3315	* @param pVM The cross context VM structure.
3316	*/
3317	VMMDECL(PGMMODE) PGMGetHostMode(PVM pVM)
3318	{
3319	switch (pVM->pgm.s.enmHostMode)
3320	{
3321	case SUPPAGINGMODE_32_BIT:
3322	case SUPPAGINGMODE_32_BIT_GLOBAL:
3323	return PGMMODE_32_BIT;
3324
3325	case SUPPAGINGMODE_PAE:
3326	case SUPPAGINGMODE_PAE_GLOBAL:
3327	return PGMMODE_PAE;
3328
3329	case SUPPAGINGMODE_PAE_NX:
3330	case SUPPAGINGMODE_PAE_GLOBAL_NX:
3331	return PGMMODE_PAE_NX;
3332
3333	case SUPPAGINGMODE_AMD64:
3334	case SUPPAGINGMODE_AMD64_GLOBAL:
3335	return PGMMODE_AMD64;
3336
3337	case SUPPAGINGMODE_AMD64_NX:
3338	case SUPPAGINGMODE_AMD64_GLOBAL_NX:
3339	return PGMMODE_AMD64_NX;
3340
3341	default: AssertMsgFailed(("enmHostMode=%d\n", pVM->pgm.s.enmHostMode)); break;
3342	}
3343
3344	return PGMMODE_INVALID;
3345	}
3346
3347
3348	/**
3349	* Get mode name.
3350	*
3351	* @returns read-only name string.
3352	* @param enmMode The mode which name is desired.
3353	*/
3354	VMMDECL(const char *) PGMGetModeName(PGMMODE enmMode)
3355	{
3356	switch (enmMode)
3357	{
3358	case PGMMODE_REAL: return "Real";
3359	case PGMMODE_PROTECTED: return "Protected";
3360	case PGMMODE_32_BIT: return "32-bit";
3361	case PGMMODE_PAE: return "PAE";
3362	case PGMMODE_PAE_NX: return "PAE+NX";
3363	case PGMMODE_AMD64: return "AMD64";
3364	case PGMMODE_AMD64_NX: return "AMD64+NX";
3365	case PGMMODE_NESTED_32BIT: return "Nested-32";
3366	case PGMMODE_NESTED_PAE: return "Nested-PAE";
3367	case PGMMODE_NESTED_AMD64: return "Nested-AMD64";
3368	case PGMMODE_EPT: return "EPT";
3369	case PGMMODE_NONE: return "None";
3370	default: return "unknown mode value";
3371	}
3372	}
3373
3374
3375	/**
3376	* Gets the physical address represented in the guest CR3 as PGM sees it.
3377	*
3378	* This is mainly for logging and debugging.
3379	*
3380	* @returns PGM's guest CR3 value.
3381	* @param pVCpu The cross context virtual CPU structure.
3382	*/
3383	VMM_INT_DECL(RTGCPHYS) PGMGetGuestCR3Phys(PVMCPU pVCpu)
3384	{
3385	return pVCpu->pgm.s.GCPhysCR3;
3386	}
3387
3388
3389
3390	/**
3391	* Notification from CPUM that the EFER.NXE bit has changed.
3392	*
3393	* @param pVCpu The cross context virtual CPU structure of the CPU for
3394	* which EFER changed.
3395	* @param fNxe The new NXE state.
3396	*/
3397	VMM_INT_DECL(void) PGMNotifyNxeChanged(PVMCPU pVCpu, bool fNxe)
3398	{
3399	/** @todo VMCPU_ASSERT_EMT_OR_NOT_RUNNING(pVCpu); */
3400	Log(("PGMNotifyNxeChanged: fNxe=%RTbool\n", fNxe));
3401
3402	pVCpu->pgm.s.fNoExecuteEnabled = fNxe;
3403	if (fNxe)
3404	{
3405	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3406	pVCpu->pgm.s.fGstPaeMbzPteMask &= ~X86_PTE_PAE_NX;
3407	pVCpu->pgm.s.fGstPaeMbzPdeMask &= ~X86_PDE_PAE_NX;
3408	pVCpu->pgm.s.fGstPaeMbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3409	/pVCpu->pgm.s.fGstPaeMbzPdpeMask - N/A /
3410	pVCpu->pgm.s.fGstAmd64MbzPteMask &= ~X86_PTE_PAE_NX;
3411	pVCpu->pgm.s.fGstAmd64MbzPdeMask &= ~X86_PDE_PAE_NX;
3412	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask &= ~X86_PDE2M_PAE_NX;
3413	pVCpu->pgm.s.fGstAmd64MbzPdpeMask &= ~X86_PDPE_LM_NX;
3414	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask &= ~X86_PDPE_LM_NX;
3415	pVCpu->pgm.s.fGstAmd64MbzPml4eMask &= ~X86_PML4E_NX;
3416
3417	pVCpu->pgm.s.fGst64ShadowedPteMask \|= X86_PTE_PAE_NX;
3418	pVCpu->pgm.s.fGst64ShadowedPdeMask \|= X86_PDE_PAE_NX;
3419	pVCpu->pgm.s.fGst64ShadowedBigPdeMask \|= X86_PDE2M_PAE_NX;
3420	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask \|= X86_PDE2M_PAE_NX;
3421	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask \|= X86_PDPE_LM_NX;
3422	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask \|= X86_PML4E_NX;
3423	}
3424	else
3425	{
3426	/pVCpu->pgm.s.fGst32BitMbzBigPdeMask - N/A /
3427	pVCpu->pgm.s.fGstPaeMbzPteMask \|= X86_PTE_PAE_NX;
3428	pVCpu->pgm.s.fGstPaeMbzPdeMask \|= X86_PDE_PAE_NX;
3429	pVCpu->pgm.s.fGstPaeMbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3430	/pVCpu->pgm.s.fGstPaeMbzPdpeMask -N/A /
3431	pVCpu->pgm.s.fGstAmd64MbzPteMask \|= X86_PTE_PAE_NX;
3432	pVCpu->pgm.s.fGstAmd64MbzPdeMask \|= X86_PDE_PAE_NX;
3433	pVCpu->pgm.s.fGstAmd64MbzBigPdeMask \|= X86_PDE2M_PAE_NX;
3434	pVCpu->pgm.s.fGstAmd64MbzPdpeMask \|= X86_PDPE_LM_NX;
3435	pVCpu->pgm.s.fGstAmd64MbzBigPdpeMask \|= X86_PDPE_LM_NX;
3436	pVCpu->pgm.s.fGstAmd64MbzPml4eMask \|= X86_PML4E_NX;
3437
3438	pVCpu->pgm.s.fGst64ShadowedPteMask &= ~X86_PTE_PAE_NX;
3439	pVCpu->pgm.s.fGst64ShadowedPdeMask &= ~X86_PDE_PAE_NX;
3440	pVCpu->pgm.s.fGst64ShadowedBigPdeMask &= ~X86_PDE2M_PAE_NX;
3441	pVCpu->pgm.s.fGst64ShadowedBigPde4PteMask &= ~X86_PDE2M_PAE_NX;
3442	pVCpu->pgm.s.fGstAmd64ShadowedPdpeMask &= ~X86_PDPE_LM_NX;
3443	pVCpu->pgm.s.fGstAmd64ShadowedPml4eMask &= ~X86_PML4E_NX;
3444	}
3445	}
3446
3447
3448	/**
3449	* Check if any pgm pool pages are marked dirty (not monitored)
3450	*
3451	* @returns bool locked/not locked
3452	* @param pVM The cross context VM structure.
3453	*/
3454	VMMDECL(bool) PGMHasDirtyPages(PVM pVM)
3455	{
3456	return pVM->pgm.s.CTX_SUFF(pPool)->cDirtyPages != 0;
3457	}
3458
3459
3460	/**
3461	* Check if this VCPU currently owns the PGM lock.
3462	*
3463	* @returns bool owner/not owner
3464	* @param pVM The cross context VM structure.
3465	*/
3466	VMMDECL(bool) PGMIsLockOwner(PVMCC pVM)
3467	{
3468	return PDMCritSectIsOwner(pVM, &pVM->pgm.s.CritSectX);
3469	}
3470
3471
3472	/**
3473	* Enable or disable large page usage
3474	*
3475	* @returns VBox status code.
3476	* @param pVM The cross context VM structure.
3477	* @param fUseLargePages Use/not use large pages
3478	*/
3479	VMMDECL(int) PGMSetLargePageUsage(PVMCC pVM, bool fUseLargePages)
3480	{
3481	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
3482
3483	pVM->pgm.s.fUseLargePages = fUseLargePages;
3484	return VINF_SUCCESS;
3485	}
3486
3487
3488	/**
3489	* Acquire the PGM lock.
3490	*
3491	* @returns VBox status code
3492	* @param pVM The cross context VM structure.
3493	* @param fVoid Set if the caller cannot handle failure returns.
3494	* @param SRC_POS The source position of the caller (RT_SRC_POS).
3495	*/
3496	#if defined(VBOX_STRICT) \|\| defined(DOXYGEN_RUNNING)
3497	int pgmLockDebug(PVMCC pVM, bool fVoid, RT_SRC_POS_DECL)
3498	#else
3499	int pgmLock(PVMCC pVM, bool fVoid)
3500	#endif
3501	{
3502	#if defined(VBOX_STRICT)
3503	int rc = PDMCritSectEnterDebug(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS, (uintptr_t)ASMReturnAddress(), RT_SRC_POS_ARGS);
3504	#else
3505	int rc = PDMCritSectEnter(pVM, &pVM->pgm.s.CritSectX, VINF_SUCCESS);
3506	#endif
3507	if (RT_SUCCESS(rc))
3508	return rc;
3509	if (fVoid)
3510	PDM_CRITSECT_RELEASE_ASSERT_RC(pVM, &pVM->pgm.s.CritSectX, rc);
3511	else
3512	AssertRC(rc);
3513	return rc;
3514	}
3515
3516
3517	/**
3518	* Release the PGM lock.
3519	*
3520	* @returns VBox status code
3521	* @param pVM The cross context VM structure.
3522	*/
3523	void pgmUnlock(PVMCC pVM)
3524	{
3525	uint32_t cDeprecatedPageLocks = pVM->pgm.s.cDeprecatedPageLocks;
3526	pVM->pgm.s.cDeprecatedPageLocks = 0;
3527	int rc = PDMCritSectLeave(pVM, &pVM->pgm.s.CritSectX);
3528	if (rc == VINF_SEM_NESTED)
3529	pVM->pgm.s.cDeprecatedPageLocks = cDeprecatedPageLocks;
3530	}
3531
3532
3533	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3534
3535	/** Format handler for PGMPAGE.
3536	* @copydoc FNRTSTRFORMATTYPE */
3537	static DECLCALLBACK(size_t) pgmFormatTypeHandlerPage(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3538	const char pszType, void const pvValue,
3539	int cchWidth, int cchPrecision, unsigned fFlags,
3540	void *pvUser)
3541	{
3542	size_t cch;
3543	PCPGMPAGE pPage = (PCPGMPAGE)pvValue;
3544	if (RT_VALID_PTR(pPage))
3545	{
3546	char szTmp[64+80];
3547
3548	cch = 0;
3549
3550	/* The single char state stuff. */
3551	static const char s_achPageStates[4] = { 'Z', 'A', 'W', 'S' };
3552	szTmp[cch++] = s_achPageStates[PGM_PAGE_GET_STATE_NA(pPage)];
3553
3554	# define IS_PART_INCLUDED(lvl) ( !(fFlags & RTSTR_F_PRECISION) \|\| cchPrecision == (lvl) \|\| cchPrecision >= (lvl)+10 )
3555	if (IS_PART_INCLUDED(5))
3556	{
3557	static const char s_achHandlerStates[4] = { '-', 't', 'w', 'a' };
3558	szTmp[cch++] = s_achHandlerStates[PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)];
3559	}
3560
3561	/* The type. */
3562	if (IS_PART_INCLUDED(4))
3563	{
3564	szTmp[cch++] = ':';
3565	static const char s_achPageTypes[8][4] = { "INV", "RAM", "MI2", "M2A", "SHA", "ROM", "MIO", "BAD" };
3566	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][0];
3567	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][1];
3568	szTmp[cch++] = s_achPageTypes[PGM_PAGE_GET_TYPE_NA(pPage)][2];
3569	}
3570
3571	/* The numbers. */
3572	if (IS_PART_INCLUDED(3))
3573	{
3574	szTmp[cch++] = ':';
3575	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_HCPHYS_NA(pPage), 16, 12, 0, RTSTR_F_ZEROPAD \| RTSTR_F_64BIT);
3576	}
3577
3578	if (IS_PART_INCLUDED(2))
3579	{
3580	szTmp[cch++] = ':';
3581	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_PAGEID(pPage), 16, 7, 0, RTSTR_F_ZEROPAD \| RTSTR_F_32BIT);
3582	}
3583
3584	if (IS_PART_INCLUDED(6))
3585	{
3586	szTmp[cch++] = ':';
3587	static const char s_achRefs[4] = { '-', 'U', '!', 'L' };
3588	szTmp[cch++] = s_achRefs[PGM_PAGE_GET_TD_CREFS_NA(pPage)];
3589	cch += RTStrFormatNumber(&szTmp[cch], PGM_PAGE_GET_TD_IDX_NA(pPage), 16, 4, 0, RTSTR_F_ZEROPAD \| RTSTR_F_16BIT);
3590	}
3591	# undef IS_PART_INCLUDED
3592
3593	cch = pfnOutput(pvArgOutput, szTmp, cch);
3594	}
3595	else
3596	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmpage-ptr>"));
3597	NOREF(pszType); NOREF(cchWidth); NOREF(pvUser);
3598	return cch;
3599	}
3600
3601
3602	/** Format handler for PGMRAMRANGE.
3603	* @copydoc FNRTSTRFORMATTYPE */
3604	static DECLCALLBACK(size_t) pgmFormatTypeHandlerRamRange(PFNRTSTROUTPUT pfnOutput, void *pvArgOutput,
3605	const char pszType, void const pvValue,
3606	int cchWidth, int cchPrecision, unsigned fFlags,
3607	void *pvUser)
3608	{
3609	size_t cch;
3610	PGMRAMRANGE const pRam = (PGMRAMRANGE const )pvValue;
3611	if (RT_VALID_PTR(pRam))
3612	{
3613	char szTmp[80];
3614	cch = RTStrPrintf(szTmp, sizeof(szTmp), "%RGp-%RGp", pRam->GCPhys, pRam->GCPhysLast);
3615	cch = pfnOutput(pvArgOutput, szTmp, cch);
3616	}
3617	else
3618	cch = pfnOutput(pvArgOutput, RT_STR_TUPLE("<bad-pgmramrange-ptr>"));
3619	NOREF(pszType); NOREF(cchWidth); NOREF(cchPrecision); NOREF(pvUser); NOREF(fFlags);
3620	return cch;
3621	}
3622
3623	/** Format type andlers to be registered/deregistered. */
3624	static const struct
3625	{
3626	char szType[24];
3627	PFNRTSTRFORMATTYPE pfnHandler;
3628	} g_aPgmFormatTypes[] =
3629	{
3630	{ "pgmpage", pgmFormatTypeHandlerPage },
3631	{ "pgmramrange", pgmFormatTypeHandlerRamRange }
3632	};
3633
3634	#endif /* !IN_R0 \|\| LOG_ENABLED */
3635
3636	/**
3637	* Registers the global string format types.
3638	*
3639	* This should be called at module load time or in some other manner that ensure
3640	* that it's called exactly one time.
3641	*
3642	* @returns IPRT status code on RTStrFormatTypeRegister failure.
3643	*/
3644	VMMDECL(int) PGMRegisterStringFormatTypes(void)
3645	{
3646	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3647	int rc = VINF_SUCCESS;
3648	unsigned i;
3649	for (i = 0; RT_SUCCESS(rc) && i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3650	{
3651	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3652	# ifdef IN_RING0
3653	if (rc == VERR_ALREADY_EXISTS)
3654	{
3655	/* in case of cleanup failure in ring-0 */
3656	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3657	rc = RTStrFormatTypeRegister(g_aPgmFormatTypes[i].szType, g_aPgmFormatTypes[i].pfnHandler, NULL);
3658	}
3659	# endif
3660	}
3661	if (RT_FAILURE(rc))
3662	while (i-- > 0)
3663	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3664
3665	return rc;
3666	#else
3667	return VINF_SUCCESS;
3668	#endif
3669	}
3670
3671
3672	/**
3673	* Deregisters the global string format types.
3674	*
3675	* This should be called at module unload time or in some other manner that
3676	* ensure that it's called exactly one time.
3677	*/
3678	VMMDECL(void) PGMDeregisterStringFormatTypes(void)
3679	{
3680	#if !defined(IN_R0) \|\| defined(LOG_ENABLED)
3681	for (unsigned i = 0; i < RT_ELEMENTS(g_aPgmFormatTypes); i++)
3682	RTStrFormatTypeDeregister(g_aPgmFormatTypes[i].szType);
3683	#endif
3684	}
3685
3686
3687	#ifdef VBOX_STRICT
3688	/**
3689	* Asserts that everything related to the guest CR3 is correctly shadowed.
3690	*
3691	* This will call PGMAssertNoMappingConflicts() and PGMAssertHandlerAndFlagsInSync(),
3692	* and assert the correctness of the guest CR3 mapping before asserting that the
3693	* shadow page tables is in sync with the guest page tables.
3694	*
3695	* @returns Number of conflicts.
3696	* @param pVM The cross context VM structure.
3697	* @param pVCpu The cross context virtual CPU structure.
3698	* @param cr3 The current guest CR3 register value.
3699	* @param cr4 The current guest CR4 register value.
3700	*/
3701	VMMDECL(unsigned) PGMAssertCR3(PVMCC pVM, PVMCPUCC pVCpu, uint64_t cr3, uint64_t cr4)
3702	{
3703	STAM_PROFILE_START(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3704
3705	uintptr_t const idxBth = pVCpu->pgm.s.idxBothModeData;
3706	AssertReturn(idxBth < RT_ELEMENTS(g_aPgmBothModeData), -VERR_PGM_MODE_IPE);
3707	AssertReturn(g_aPgmBothModeData[idxBth].pfnAssertCR3, -VERR_PGM_MODE_IPE);
3708
3709	PGM_LOCK_VOID(pVM);
3710	unsigned cErrors = g_aPgmBothModeData[idxBth].pfnAssertCR3(pVCpu, cr3, cr4, 0, ~(RTGCPTR)0);
3711	PGM_UNLOCK(pVM);
3712
3713	STAM_PROFILE_STOP(&pVCpu->pgm.s.Stats.CTX_MID_Z(Stat,SyncCR3), a);
3714	return cErrors;
3715	}
3716	#endif /* VBOX_STRICT */
3717

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAll.cpp@ 91854

Download in other formats: