PGMAllPhys.cpp

Last change on this file was 108843, checked in by vboxsync, 5 weeks ago
VMM/PGM,NEM: Some early page table management infrastructure for ARMv8, bugref:10388
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 212.5 KB

Line
1	/* $Id: PGMAllPhys.cpp 108843 2025-04-04 08:36:32Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2024 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.215389.xyz.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_PHYS
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#ifdef IN_RING0
35	# define VBOX_VMM_TARGET_X86
36	#endif
37	#include <VBox/vmm/pgm.h>
38	#include <VBox/vmm/trpm.h>
39	#include <VBox/vmm/vmm.h>
40	#include <VBox/vmm/iem.h>
41	#include <VBox/vmm/iom.h>
42	#include <VBox/vmm/em.h>
43	#include <VBox/vmm/nem.h>
44	#include "PGMInternal.h"
45	#include <VBox/vmm/vmcc.h>
46	#include "PGMInline.h"
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49	#include <iprt/assert.h>
50	#include <iprt/string.h>
51	#include <VBox/log.h>
52	#ifdef IN_RING3
53	# include <iprt/thread.h>
54	# ifdef VBOX_WITH_ONLY_PGM_NEM_MODE
55	# include <iprt/zero.h>
56	# endif
57	#elif defined(IN_RING0)
58	# include <iprt/mem.h>
59	# include <iprt/memobj.h>
60	#endif
61
62
63	/*********************************************************************************************************************************
64	* Defined Constants And Macros *
65	*********************************************************************************************************************************/
66	/** Enable the physical TLB. */
67	#define PGM_WITH_PHYS_TLB
68
69	/** @def PGM_HANDLER_PHYS_IS_VALID_STATUS
70	* Checks if valid physical access handler return code (normal handler, not PF).
71	*
72	* Checks if the given strict status code is one of the expected ones for a
73	* physical access handler in the current context.
74	*
75	* @returns true or false.
76	* @param a_rcStrict The status code.
77	* @param a_fWrite Whether it is a write or read being serviced.
78	*
79	* @remarks We wish to keep the list of statuses here as short as possible.
80	* When changing, please make sure to update the PGMPhysRead,
81	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
82	*/
83	#ifdef IN_RING3
84	# define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
85	( (a_rcStrict) == VINF_SUCCESS \
86	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
87	#elif defined(IN_RING0)
88	#define PGM_HANDLER_PHYS_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
89	( (a_rcStrict) == VINF_SUCCESS \
90	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT \
91	\
92	\|\| (a_rcStrict) == ((a_fWrite) ? VINF_IOM_R3_MMIO_WRITE : VINF_IOM_R3_MMIO_READ) \
93	\|\| (a_rcStrict) == VINF_IOM_R3_MMIO_READ_WRITE \
94	\|\| ((a_rcStrict) == VINF_IOM_R3_MMIO_COMMIT_WRITE && (a_fWrite)) \
95	\
96	\|\| (a_rcStrict) == VINF_EM_RAW_EMULATE_INSTR \
97	\|\| (a_rcStrict) == VINF_EM_DBG_STOP \
98	\|\| (a_rcStrict) == VINF_EM_DBG_EVENT \
99	\|\| (a_rcStrict) == VINF_EM_DBG_BREAKPOINT \
100	\|\| (a_rcStrict) == VINF_EM_OFF \
101	\|\| (a_rcStrict) == VINF_EM_SUSPEND \
102	\|\| (a_rcStrict) == VINF_EM_RESET \
103	)
104	#else
105	# error "Context?"
106	#endif
107
108	/** @def PGM_HANDLER_VIRT_IS_VALID_STATUS
109	* Checks if valid virtual access handler return code (normal handler, not PF).
110	*
111	* Checks if the given strict status code is one of the expected ones for a
112	* virtual access handler in the current context.
113	*
114	* @returns true or false.
115	* @param a_rcStrict The status code.
116	* @param a_fWrite Whether it is a write or read being serviced.
117	*
118	* @remarks We wish to keep the list of statuses here as short as possible.
119	* When changing, please make sure to update the PGMPhysRead,
120	* PGMPhysWrite, PGMPhysReadGCPtr and PGMPhysWriteGCPtr docs too.
121	*/
122	#ifdef IN_RING3
123	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
124	( (a_rcStrict) == VINF_SUCCESS \
125	\|\| (a_rcStrict) == VINF_PGM_HANDLER_DO_DEFAULT)
126	#elif defined(IN_RING0)
127	# define PGM_HANDLER_VIRT_IS_VALID_STATUS(a_rcStrict, a_fWrite) \
128	(false /* no virtual handlers in ring-0! */ )
129	#else
130	# error "Context?"
131	#endif
132
133
134
135	/**
136	* Calculate the actual table size.
137	*
138	* The memory is layed out like this:
139	* - PGMPHYSHANDLERTREE (8 bytes)
140	* - Allocation bitmap (8-byte size align)
141	* - Slab of PGMPHYSHANDLER. Start is 64 byte aligned.
142	*/
143	uint32_t pgmHandlerPhysicalCalcTableSizes(uint32_t pcEntries, uint32_t pcbTreeAndBitmap)
144	{
145	/*
146	* A minimum of 64 entries and a maximum of ~64K.
147	*/
148	uint32_t cEntries = *pcEntries;
149	if (cEntries <= 64)
150	cEntries = 64;
151	else if (cEntries >= _64K)
152	cEntries = _64K;
153	else
154	cEntries = RT_ALIGN_32(cEntries, 16);
155
156	/*
157	* Do the initial calculation.
158	*/
159	uint32_t cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
160	uint32_t cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
161	uint32_t cbTable = cEntries * sizeof(PGMPHYSHANDLER);
162	uint32_t cbTotal = cbTreeAndBitmap + cbTable;
163
164	/*
165	* Align the total and try use up extra space from that.
166	*/
167	uint32_t cbTotalAligned = RT_ALIGN_32(cbTotal, RT_MAX(HOST_PAGE_SIZE_DYNAMIC, _16K));
168	uint32_t cAvail = cbTotalAligned - cbTotal;
169	cAvail /= sizeof(PGMPHYSHANDLER);
170	if (cAvail >= 1)
171	for (;;)
172	{
173	cbBitmap = RT_ALIGN_32(cEntries, 64) / 8;
174	cbTreeAndBitmap = RT_ALIGN_32(sizeof(PGMPHYSHANDLERTREE) + cbBitmap, 64);
175	cbTable = cEntries * sizeof(PGMPHYSHANDLER);
176	cbTotal = cbTreeAndBitmap + cbTable;
177	if (cbTotal <= cbTotalAligned)
178	break;
179	cEntries--;
180	Assert(cEntries >= 16);
181	}
182
183	/*
184	* Return the result.
185	*/
186	*pcbTreeAndBitmap = cbTreeAndBitmap;
187	*pcEntries = cEntries;
188	return cbTotalAligned;
189	}
190
191
192
193	/*********************************************************************************************************************************
194	* Access Handlers for ROM and MMIO2 *
195	*********************************************************************************************************************************/
196
197	#ifndef IN_RING3
198
199	/**
200	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
201	* \#PF access handler callback for guest ROM range write access.}
202	*
203	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
204	*/
205	DECLCALLBACK(VBOXSTRICTRC) pgmPhysRomWritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
206	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
207
208	{
209	AssertReturn(uUser < RT_ELEMENTS(pVM->pgmr0.s.apRomRanges), VINF_EM_RAW_EMULATE_INSTR);
210	PPGMROMRANGE const pRom = pVM->pgmr0.s.apRomRanges[uUser];
211	AssertReturn(pRom, VINF_EM_RAW_EMULATE_INSTR);
212
213	uint32_t const iPage = (GCPhysFault - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
214	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_3);
215	#ifdef IN_RING0
216	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
217	#endif
218
219	RT_NOREF(uErrorCode, pvFault);
220	Assert(uErrorCode & X86_TRAP_PF_RW); /* This shall not be used for read access! */
221
222	int rc;
223	switch (pRom->aPages[iPage].enmProt)
224	{
225	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
226	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
227	{
228	/*
229	* If it's a simple instruction which doesn't change the cpu state
230	* we will simply skip it. Otherwise we'll have to defer it to REM.
231	*/
232	uint32_t cbOp;
233	PDISSTATE pDis = &pVCpu->pgm.s.Dis;
234	rc = EMInterpretDisasCurrent(pVCpu, pDis, &cbOp);
235	if ( RT_SUCCESS(rc)
236	&& pDis->uCpuMode == DISCPUMODE_32BIT /** @todo why does this matter? */
237	&& !(pDis->x86.fPrefix & (DISPREFIX_REPNE \| DISPREFIX_REP \| DISPREFIX_SEG)))
238	{
239	switch (pDis->x86.bOpCode)
240	{
241	/** @todo Find other instructions we can safely skip, possibly
242	* adding this kind of detection to DIS or EM. */
243	case OP_MOV:
244	pCtx->rip += cbOp;
245	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteHandled);
246	return VINF_SUCCESS;
247	}
248	}
249	break;
250	}
251
252	case PGMROMPROT_READ_RAM_WRITE_RAM:
253	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
254	rc = PGMHandlerPhysicalPageTempOff(pVM, pRom->GCPhys, GCPhysFault & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
255	AssertRC(rc);
256	break; /** @todo Must edit the shadow PT and restart the instruction, not use the interpreter! */
257
258	case PGMROMPROT_READ_ROM_WRITE_RAM:
259	/* Handle it in ring-3 because it's way easier there. */
260	pRom->aPages[iPage].LiveSave.fWrittenTo = true;
261	break;
262
263	default:
264	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhysFault=%RGp\n",
265	pRom->aPages[iPage].enmProt, iPage, GCPhysFault),
266	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
267	}
268
269	STAM_COUNTER_INC(&pVCpu->pgm.s.Stats.StatRZGuestROMWriteUnhandled);
270	return VINF_EM_RAW_EMULATE_INSTR;
271	}
272
273	#endif /* !IN_RING3 */
274
275
276	/**
277	* @callback_method_impl{FNPGMPHYSHANDLER,
278	* Access handler callback for ROM write accesses.}
279	*
280	* @remarks The @a uUser argument is the PGMROMRANGE::GCPhys value.
281	*/
282	DECLCALLBACK(VBOXSTRICTRC)
283	pgmPhysRomWriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
284	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
285	{
286	AssertReturn(uUser < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges), VERR_INTERNAL_ERROR_3);
287	PPGMROMRANGE const pRom = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges[uUser];
288	AssertReturn(pRom, VERR_INTERNAL_ERROR_3);
289
290	uint32_t const iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
291	AssertReturn(iPage < (pRom->cb >> GUEST_PAGE_SHIFT), VERR_INTERNAL_ERROR_2);
292	#ifdef IN_RING0
293	AssertReturn(iPage < pVM->pgmr0.s.acRomRangePages[uUser], VERR_INTERNAL_ERROR_2);
294	#endif
295	PPGMROMPAGE const pRomPage = &pRom->aPages[iPage];
296
297	Log5(("pgmPhysRomWriteHandler: %d %c %#08RGp %#04zx\n", pRomPage->enmProt, enmAccessType == PGMACCESSTYPE_READ ? 'R' : 'W', GCPhys, cbBuf));
298	RT_NOREF(pVCpu, pvPhys, enmOrigin);
299
300	if (enmAccessType == PGMACCESSTYPE_READ)
301	{
302	switch (pRomPage->enmProt)
303	{
304	/*
305	* Take the default action.
306	*/
307	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
308	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
309	case PGMROMPROT_READ_ROM_WRITE_RAM:
310	case PGMROMPROT_READ_RAM_WRITE_RAM:
311	return VINF_PGM_HANDLER_DO_DEFAULT;
312
313	default:
314	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
315	pRom->aPages[iPage].enmProt, iPage, GCPhys),
316	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
317	}
318	}
319	else
320	{
321	Assert(enmAccessType == PGMACCESSTYPE_WRITE);
322	switch (pRomPage->enmProt)
323	{
324	/*
325	* Ignore writes.
326	*/
327	case PGMROMPROT_READ_ROM_WRITE_IGNORE:
328	case PGMROMPROT_READ_RAM_WRITE_IGNORE:
329	return VINF_SUCCESS;
330
331	/*
332	* Write to the RAM page.
333	*/
334	case PGMROMPROT_READ_ROM_WRITE_RAM:
335	case PGMROMPROT_READ_RAM_WRITE_RAM: /* yes this will get here too, it's way simpler that way. */
336	{
337	/* This should be impossible now, pvPhys doesn't work cross page anylonger. */
338	Assert(((GCPhys - pRom->GCPhys + cbBuf - 1) >> GUEST_PAGE_SHIFT) == iPage);
339
340	/*
341	* Take the lock, do lazy allocation, map the page and copy the data.
342	*
343	* Note that we have to bypass the mapping TLB since it works on
344	* guest physical addresses and entering the shadow page would
345	* kind of screw things up...
346	*/
347	PGM_LOCK_VOID(pVM);
348
349	PPGMPAGE pShadowPage = &pRomPage->Shadow;
350	if (!PGMROMPROT_IS_ROM(pRomPage->enmProt))
351	{
352	pShadowPage = pgmPhysGetPage(pVM, GCPhys);
353	AssertLogRelMsgReturnStmt(pShadowPage, ("%RGp\n", GCPhys), PGM_UNLOCK(pVM), VERR_PGM_PHYS_PAGE_GET_IPE);
354	}
355
356	void *pvDstPage;
357	int rc;
358	#if defined(VBOX_WITH_PGM_NEM_MODE) && defined(IN_RING3)
359	if (PGM_IS_IN_NEM_MODE(pVM) && PGMROMPROT_IS_ROM(pRomPage->enmProt))
360	{
361	pvDstPage = &pRom->pbR3Alternate[GCPhys - pRom->GCPhys];
362	rc = VINF_SUCCESS;
363	}
364	else
365	#endif
366	{
367	rc = pgmPhysPageMakeWritableAndMap(pVM, pShadowPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK, &pvDstPage);
368	if (RT_SUCCESS(rc))
369	pvDstPage = (uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK);
370	}
371	if (RT_SUCCESS(rc))
372	{
373	memcpy((uint8_t *)pvDstPage + (GCPhys & GUEST_PAGE_OFFSET_MASK), pvBuf, cbBuf);
374	pRomPage->LiveSave.fWrittenTo = true;
375
376	AssertMsg( rc == VINF_SUCCESS
377	\|\| ( rc == VINF_PGM_SYNC_CR3
378	&& VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
379	, ("%Rrc\n", rc));
380	rc = VINF_SUCCESS;
381	}
382
383	PGM_UNLOCK(pVM);
384	return rc;
385	}
386
387	default:
388	AssertMsgFailedReturn(("enmProt=%d iPage=%d GCPhys=%RGp\n",
389	pRom->aPages[iPage].enmProt, iPage, GCPhys),
390	VERR_IPE_NOT_REACHED_DEFAULT_CASE);
391	}
392	}
393	}
394
395
396	/**
397	* Common worker for pgmPhysMmio2WriteHandler and pgmPhysMmio2WritePfHandler.
398	*/
399	static VBOXSTRICTRC pgmPhysMmio2WriteHandlerCommon(PVMCC pVM, PVMCPUCC pVCpu, uint64_t hMmio2, RTGCPHYS GCPhys, RTGCPTR GCPtr)
400	{
401	/*
402	* Get the MMIO2 range.
403	*/
404	AssertReturn(hMmio2 < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_3);
405	AssertReturn(hMmio2 != 0, VERR_INTERNAL_ERROR_3);
406	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[hMmio2 - 1];
407	Assert(pMmio2->idMmio2 == hMmio2);
408	AssertReturn((pMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES) == PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES,
409	VERR_INTERNAL_ERROR_4);
410
411	/*
412	* Get the page and make sure it's an MMIO2 page.
413	*/
414	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
415	AssertReturn(pPage, VINF_EM_RAW_EMULATE_INSTR);
416	AssertReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2, VINF_EM_RAW_EMULATE_INSTR);
417
418	/*
419	* Set the dirty flag so we can avoid scanning all the pages when it isn't dirty.
420	* (The PGM_PAGE_HNDL_PHYS_STATE_DISABLED handler state indicates that a single
421	* page is dirty, saving the need for additional storage (bitmap).)
422	*/
423	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_IS_DIRTY;
424
425	/*
426	* Disable the handler for this page.
427	*/
428	int rc = PGMHandlerPhysicalPageTempOff(pVM, pMmio2->GCPhys, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
429	AssertRC(rc);
430	#ifndef IN_RING3
431	if (RT_SUCCESS(rc) && GCPtr != ~(RTGCPTR)0)
432	{
433	rc = PGMShwMakePageWritable(pVCpu, GCPtr, PGM_MK_PG_IS_MMIO2 \| PGM_MK_PG_IS_WRITE_FAULT);
434	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VERR_PAGE_TABLE_NOT_PRESENT,
435	("PGMShwModifyPage -> GCPtr=%RGv rc=%d\n", GCPtr, rc), rc);
436	}
437	#else
438	RT_NOREF(pVCpu, GCPtr);
439	#endif
440	return VINF_SUCCESS;
441	}
442
443
444	#ifndef IN_RING3
445	/**
446	* @callback_method_impl{FNPGMRZPHYSPFHANDLER,
447	* \#PF access handler callback for guest MMIO2 dirty page tracing.}
448	*
449	* @remarks The @a uUser is the MMIO2 index.
450	*/
451	DECLCALLBACK(VBOXSTRICTRC) pgmPhysMmio2WritePfHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCUINT uErrorCode, PCPUMCTX pCtx,
452	RTGCPTR pvFault, RTGCPHYS GCPhysFault, uint64_t uUser)
453	{
454	RT_NOREF(pVCpu, uErrorCode, pCtx);
455	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
456	if (RT_SUCCESS(rcStrict))
457	{
458	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhysFault, pvFault);
459	PGM_UNLOCK(pVM);
460	}
461	return rcStrict;
462	}
463	#endif /* !IN_RING3 */
464
465
466	/**
467	* @callback_method_impl{FNPGMPHYSHANDLER,
468	* Access handler callback for MMIO2 dirty page tracing.}
469	*
470	* @remarks The @a uUser is the MMIO2 index.
471	*/
472	DECLCALLBACK(VBOXSTRICTRC)
473	pgmPhysMmio2WriteHandler(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, void pvPhys, void pvBuf, size_t cbBuf,
474	PGMACCESSTYPE enmAccessType, PGMACCESSORIGIN enmOrigin, uint64_t uUser)
475	{
476	VBOXSTRICTRC rcStrict = PGM_LOCK(pVM); /* We should already have it, but just make sure we do. */
477	if (RT_SUCCESS(rcStrict))
478	{
479	rcStrict = pgmPhysMmio2WriteHandlerCommon(pVM, pVCpu, uUser, GCPhys, ~(RTGCPTR)0);
480	PGM_UNLOCK(pVM);
481	if (rcStrict == VINF_SUCCESS)
482	rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
483	}
484	RT_NOREF(pvPhys, pvBuf, cbBuf, enmAccessType, enmOrigin);
485	return rcStrict;
486	}
487
488
489
490	/*********************************************************************************************************************************
491	* RAM Ranges *
492	*********************************************************************************************************************************/
493
494	#ifdef VBOX_STRICT
495	/**
496	* Asserts that the RAM range structures are sane.
497	*/
498	DECLHIDDEN(bool) pgmPhysAssertRamRangesLocked(PVMCC pVM, bool fInUpdate, bool fRamRelaxed)
499	{
500	bool fRet = true;
501
502	/*
503	* Check the generation ID. This is stable since we own the PGM lock.
504	*/
505	AssertStmt((pVM->pgm.s.RamRangeUnion.idGeneration & 1U) == (unsigned)fInUpdate, fRet = false);
506
507	/*
508	* Check the entry count and max ID.
509	*/
510	uint32_t const idRamRangeMax = pVM->pgm.s.idRamRangeMax;
511	/* Since this is set to the highest ID, it cannot be the same as the table size. */
512	AssertStmt(idRamRangeMax < RT_ELEMENTS(pVM->pgm.s.apRamRanges), fRet = false);
513
514	/* Because ID=0 is reserved, it's one less than the table size and at most the
515	same as the max ID. */
516	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
517	AssertStmt(cLookupEntries < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), fRet = false);
518	AssertStmt(cLookupEntries <= idRamRangeMax, fRet = false);
519
520	/*
521	* Check the pointer table(s).
522	*/
523	/* The first entry shall be empty. */
524	AssertStmt(pVM->pgm.s.apRamRanges[0] == NULL, fRet = false);
525	# ifdef IN_RING0
526	AssertStmt(pVM->pgmr0.s.apRamRanges[0] == NULL, fRet = false);
527	AssertStmt(pVM->pgmr0.s.acRamRangePages[0] == 0, fRet = false);
528	# endif
529
530	uint32_t cMappedRanges = 0;
531	for (uint32_t idRamRange = 1; idRamRange <= idRamRangeMax; idRamRange++)
532	{
533	# ifdef IN_RING0
534	PPGMRAMRANGE const pRamRange = pVM->pgmr0.s.apRamRanges[idRamRange];
535	AssertContinueStmt(pRamRange, fRet = false);
536	AssertStmt(pVM->pgm.s.apRamRanges[idRamRange] != NIL_RTR3PTR, fRet = false);
537	AssertStmt( (pRamRange->cb >> GUEST_PAGE_SHIFT) == pVM->pgmr0.s.acRamRangePages[idRamRange]
538	\|\| ( (pRamRange->cb >> GUEST_PAGE_SHIFT) < pVM->pgmr0.s.acRamRangePages[idRamRange]
539	&& !(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX)),
540	fRet = false);
541	# else
542	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apRamRanges[idRamRange];
543	AssertContinueStmt(pRamRange, fRet = false);
544	# endif
545	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
546	if (pRamRange->GCPhys != NIL_RTGCPHYS)
547	{
548	cMappedRanges++;
549	AssertStmt((pRamRange->GCPhys & GUEST_PAGE_OFFSET_MASK) == 0, fRet = false);
550	AssertStmt((pRamRange->GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK, fRet = false);
551	AssertStmt(pRamRange->GCPhysLast > pRamRange->GCPhys, fRet = false);
552	AssertStmt(pRamRange->GCPhysLast - pRamRange->GCPhys + 1U == pRamRange->cb, fRet = false);
553	}
554	else
555	{
556	AssertStmt(pRamRange->GCPhysLast == NIL_RTGCPHYS, fRet = false);
557	AssertStmt(PGM_RAM_RANGE_IS_AD_HOC(pRamRange) \|\| fRamRelaxed, fRet = false);
558	}
559	}
560
561	/*
562	* Check that the lookup table is sorted and contains the right information.
563	*/
564	AssertMsgStmt(cMappedRanges == cLookupEntries,
565	("cMappedRanges=%#x cLookupEntries=%#x\n", cMappedRanges, cLookupEntries),
566	fRet = false);
567	RTGCPHYS GCPhysPrev = ~(RTGCPHYS)0;
568	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
569	{
570	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
571	AssertContinueStmt(idRamRange > 0 && idRamRange <= idRamRangeMax, fRet = false);
572	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm,pgmr0,pgmrc).s.apRamRanges[idRamRange];
573	AssertContinueStmt(pRamRange, fRet = false);
574
575	AssertStmt(pRamRange->idRange == idRamRange, fRet = false);
576	AssertStmt(pRamRange->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]),
577	fRet = false);
578	AssertStmt(pRamRange->GCPhysLast == pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast, fRet = false);
579
580	AssertStmt(pRamRange->GCPhys >= GCPhysPrev + 1U, fRet = false);
581	GCPhysPrev = pRamRange->GCPhysLast;
582	}
583
584	return fRet;
585	}
586	#endif /* VBOX_STRICT */
587
588
589	/**
590	* Invalidates the RAM range TLBs.
591	*
592	* @param pVM The cross context VM structure.
593	*/
594	void pgmPhysInvalidRamRangeTlbs(PVMCC pVM)
595	{
596	PGM_LOCK_VOID(pVM);
597
598	/* This is technically only required when freeing the PCNet MMIO2 range
599	during ancient saved state loading. The code freeing the RAM range
600	will make sure this function is called in both rings. */
601	RT_ZERO(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb);
602	VMCC_FOR_EACH_VMCPU_STMT(pVM, RT_ZERO(pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb));
603
604	PGM_UNLOCK(pVM);
605	}
606
607
608	/**
609	* Tests if a value of type RTGCPHYS is negative if the type had been signed
610	* instead of unsigned.
611	*
612	* @returns @c true if negative, @c false if positive or zero.
613	* @param a_GCPhys The value to test.
614	* @todo Move me to iprt/types.h.
615	*/
616	#define RTGCPHYS_IS_NEGATIVE(a_GCPhys) ((a_GCPhys) & ((RTGCPHYS)1 << (sizeof(RTGCPHYS)*8 - 1)))
617
618
619	/**
620	* Slow worker for pgmPhysGetRange.
621	*
622	* @copydoc pgmPhysGetRange
623	* @note Caller owns the PGM lock.
624	*/
625	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeSlow(PVMCC pVM, RTGCPHYS GCPhys)
626	{
627	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
628
629	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
630	uint32_t idxStart = 0;
631	for (;;)
632	{
633	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
634	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
635	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
636	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
637	if (off <= cbEntryMinus1)
638	{
639	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
640	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
641	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
642	Assert(pRamRange);
643	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
644	return pRamRange;
645	}
646	if (RTGCPHYS_IS_NEGATIVE(off))
647	{
648	if (idxStart < idxLookup)
649	idxEnd = idxLookup;
650	else
651	break;
652	}
653	else
654	{
655	idxLookup += 1;
656	if (idxLookup < idxEnd)
657	idxStart = idxLookup;
658	else
659	break;
660	}
661	}
662	return NULL;
663	}
664
665
666	/**
667	* Slow worker for pgmPhysGetRangeAtOrAbove.
668	*
669	* @copydoc pgmPhysGetRangeAtOrAbove
670	*/
671	DECLHIDDEN(PPGMRAMRANGE) pgmPhysGetRangeAtOrAboveSlow(PVMCC pVM, RTGCPHYS GCPhys)
672	{
673	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
674
675	uint32_t idRamRangeLastLeft = UINT32_MAX;
676	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
677	uint32_t idxStart = 0;
678	for (;;)
679	{
680	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
681	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
682	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
683	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
684	if (off <= cbEntryMinus1)
685	{
686	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
687	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
688	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
689	Assert(pRamRange);
690	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
691	return pRamRange;
692	}
693	if (RTGCPHYS_IS_NEGATIVE(off))
694	{
695	idRamRangeLastLeft = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
696	if (idxStart < idxLookup)
697	idxEnd = idxLookup;
698	else
699	break;
700	}
701	else
702	{
703	idxLookup += 1;
704	if (idxLookup < idxEnd)
705	idxStart = idxLookup;
706	else
707	break;
708	}
709	}
710	if (idRamRangeLastLeft != UINT32_MAX)
711	{
712	AssertReturn(idRamRangeLastLeft < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
713	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRangeLastLeft];
714	Assert(pRamRange);
715	return pRamRange;
716	}
717	return NULL;
718	}
719
720
721	/**
722	* Slow worker for pgmPhysGetPage.
723	*
724	* @copydoc pgmPhysGetPage
725	*/
726	DECLHIDDEN(PPGMPAGE) pgmPhysGetPageSlow(PVMCC pVM, RTGCPHYS GCPhys)
727	{
728	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
729
730	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
731	uint32_t idxStart = 0;
732	for (;;)
733	{
734	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
735	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
736	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
737	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
738	if (off <= cbEntryMinus1)
739	{
740	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
741	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), NULL);
742	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
743	AssertReturn(pRamRange, NULL);
744	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
745
746	/* Get the page. */
747	Assert(off < pRamRange->cb);
748	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
749	#ifdef IN_RING0
750	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], NULL);
751	#endif
752	return &pRamRange->aPages[idxPage];
753	}
754	if (RTGCPHYS_IS_NEGATIVE(off))
755	{
756	if (idxStart < idxLookup)
757	idxEnd = idxLookup;
758	else
759	break;
760	}
761	else
762	{
763	idxLookup += 1;
764	if (idxLookup < idxEnd)
765	idxStart = idxLookup;
766	else
767	break;
768	}
769	}
770	return NULL;
771	}
772
773
774	/**
775	* Slow worker for pgmPhysGetPageEx.
776	*
777	* @copydoc pgmPhysGetPageEx
778	*/
779	DECLHIDDEN(int) pgmPhysGetPageExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage)
780	{
781	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
782
783	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
784	uint32_t idxStart = 0;
785	for (;;)
786	{
787	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
788	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
789	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
790	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
791	if (off <= cbEntryMinus1)
792	{
793	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
794	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
795	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
796	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
797	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
798
799	/* Get the page. */
800	Assert(off < pRamRange->cb);
801	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
802	#ifdef IN_RING0
803	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
804	#endif
805	*ppPage = &pRamRange->aPages[idxPage];
806	return VINF_SUCCESS;
807	}
808	if (RTGCPHYS_IS_NEGATIVE(off))
809	{
810	if (idxStart < idxLookup)
811	idxEnd = idxLookup;
812	else
813	break;
814	}
815	else
816	{
817	idxLookup += 1;
818	if (idxLookup < idxEnd)
819	idxStart = idxLookup;
820	else
821	break;
822	}
823	}
824
825	*ppPage = NULL;
826	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
827	}
828
829
830	/**
831	* Slow worker for pgmPhysGetPageAndRangeEx.
832	*
833	* @copydoc pgmPhysGetPageAndRangeEx
834	*/
835	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlow(PVMCC pVM, RTGCPHYS GCPhys, PPPGMPAGE ppPage, PPGMRAMRANGE *ppRam)
836	{
837	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,RamRangeTlbMisses));
838
839	uint32_t idxEnd = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
840	uint32_t idxStart = 0;
841	for (;;)
842	{
843	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
844	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
845	RTGCPHYS const cbEntryMinus1 = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast - GCPhysEntryFirst;
846	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
847	if (off <= cbEntryMinus1)
848	{
849	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
850	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_PGM_PHYS_RAM_LOOKUP_IPE);
851	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
852	AssertReturn(pRamRange, VERR_PGM_PHYS_RAM_LOOKUP_IPE);
853	pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
854
855	/* Get the page. */
856	Assert(off < pRamRange->cb);
857	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
858	#ifdef IN_RING0
859	AssertReturn(idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange], VERR_PGM_PHYS_RAM_LOOKUP_IPE);
860	#endif
861	*ppRam = pRamRange;
862	*ppPage = &pRamRange->aPages[idxPage];
863	return VINF_SUCCESS;
864	}
865	if (RTGCPHYS_IS_NEGATIVE(off))
866	{
867	if (idxStart < idxLookup)
868	idxEnd = idxLookup;
869	else
870	break;
871	}
872	else
873	{
874	idxLookup += 1;
875	if (idxLookup < idxEnd)
876	idxStart = idxLookup;
877	else
878	break;
879	}
880	}
881
882	*ppRam = NULL;
883	*ppPage = NULL;
884	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
885	}
886
887
888	/**
889	* Slow worker for pgmPhysGetPageAndRangeExLockless.
890	*
891	* @copydoc pgmPhysGetPageAndRangeExLockless
892	*/
893	DECLHIDDEN(int) pgmPhysGetPageAndRangeExSlowLockless(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys,
894	PGMPAGE volatile ppPage, PGMRAMRANGE volatile ppRam)
895	{
896	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbMisses));
897
898	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT RamRangeUnion;
899	RamRangeUnion.u64Combined = ASMAtomicUoReadU64(&pVM->pgm.s.RamRangeUnion.u64Combined);
900
901	uint32_t idxEnd = RT_MIN(RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
902	uint32_t idxStart = 0;
903	for (;;)
904	{
905	/* Read the entry as atomically as possible: */
906	uint32_t idxLookup = idxStart + (idxEnd - idxStart) / 2;
907	PGMRAMRANGELOOKUPENTRY Entry;
908	#if (RTASM_HAVE_READ_U128+0) & 1
909	Entry.u128Normal = ASMAtomicUoReadU128U(&pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile);
910	#else
911	Entry.u128Normal.s.Lo = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo;
912	Entry.u128Normal.s.Hi = pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Hi;
913	ASMCompilerBarrier(); /paranoia^2/
914	if (RT_LIKELY(Entry.u128Normal.s.Lo == pVM->pgm.s.aRamRangeLookup[idxLookup].u128Volatile.s.Lo))
915	{ /* likely */ }
916	else
917	break;
918	#endif
919
920	/* Check how GCPhys relates to the entry: */
921	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(Entry);
922	RTGCPHYS const cbEntryMinus1 = Entry.GCPhysLast - GCPhysEntryFirst;
923	RTGCPHYS const off = GCPhys - GCPhysEntryFirst;
924	if (off <= cbEntryMinus1)
925	{
926	/* We seem to have a match. If, however, anything doesn't match up
927	bail and redo owning the lock. No asserting here as we may be
928	racing removal/insertion. */
929	if (!RTGCPHYS_IS_NEGATIVE(off))
930	{
931	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(Entry);
932	if (idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
933	{
934	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
935	if (pRamRange)
936	{
937	if ( pRamRange->GCPhys == GCPhysEntryFirst
938	&& pRamRange->cb == cbEntryMinus1 + 1U)
939	{
940	RTGCPHYS const idxPage = off >> GUEST_PAGE_SHIFT;
941	#ifdef IN_RING0
942	if (idxPage < pVM->pgmr0.s.acRamRangePages[idRamRange])
943	#endif
944	{
945	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = pRamRange;
946	*ppRam = pRamRange;
947	*ppPage = &pRamRange->aPages[idxPage];
948	return VINF_SUCCESS;
949	}
950	}
951	}
952	}
953	}
954	break;
955	}
956	if (RTGCPHYS_IS_NEGATIVE(off))
957	{
958	if (idxStart < idxLookup)
959	idxEnd = idxLookup;
960	else
961	break;
962	}
963	else
964	{
965	idxLookup += 1;
966	if (idxLookup < idxEnd)
967	idxStart = idxLookup;
968	else
969	break;
970	}
971	}
972
973	/*
974	* If we get down here, we do the lookup again but while owning the PGM lock.
975	*/
976	*ppRam = NULL;
977	*ppPage = NULL;
978	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,RamRangeTlbLocking));
979
980	PGM_LOCK_VOID(pVM);
981	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, (PPGMPAGE )ppPage, (PPGMRAMRANGE )ppRam);
982	PGM_UNLOCK(pVM);
983
984	PGMRAMRANGE volatile * const pRam = *ppRam;
985	if (pRam)
986	pVCpu->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRangesTlb[PGM_RAMRANGE_TLB_IDX(GCPhys)] = (PPGMRAMRANGE)pRam;
987	return rc;
988	}
989
990
991	/**
992	* Common worker for pgmR3PhysAllocateRamRange, PGMR0PhysAllocateRamRangeReq,
993	* and pgmPhysMmio2RegisterWorker2.
994	*/
995	DECLHIDDEN(int) pgmPhysRamRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint32_t fFlags, uint32_t *pidNewRange)
996	{
997
998	/*
999	* Allocate the RAM range structure and map it into ring-3.
1000	*/
1001	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1002	#ifdef IN_RING0
1003	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1004	int rc = RTR0MemObjAllocPage(&hMemObj, cbRamRange, false /fExecutable/);
1005	#else
1006	PPGMRAMRANGE pRamRange;
1007	int rc = SUPR3PageAlloc(cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC, 0 /fFlags/, (void **)&pRamRange);
1008	#endif
1009	if (RT_SUCCESS(rc))
1010	{
1011	/* Zero the memory and do basic range init before mapping it into userland. */
1012	#ifdef IN_RING0
1013	PPGMRAMRANGE const pRamRange = (PPGMRAMRANGE)RTR0MemObjAddress(hMemObj);
1014	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1015	#endif
1016	RT_BZERO(pRamRange, cbRamRange);
1017
1018	pRamRange->GCPhys = NIL_RTGCPHYS;
1019	pRamRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1020	pRamRange->GCPhysLast = NIL_RTGCPHYS;
1021	pRamRange->fFlags = fFlags;
1022	pRamRange->idRange = UINT32_MAX / 2;
1023
1024	#ifdef IN_RING0
1025	/* Map it into userland. */
1026	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1027	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1028	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1029	if (RT_SUCCESS(rc))
1030	#endif
1031	{
1032	/*
1033	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1034	*/
1035	rc = PGM_LOCK(pVM);
1036	if (RT_SUCCESS(rc))
1037	{
1038	/*
1039	* Allocate a range ID.
1040	*/
1041	uint32_t idRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax + 1;
1042	if (idRamRange != 0 && idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges))
1043	{
1044	#ifdef IN_RING0
1045	if (pVM->pgmr0.s.apRamRanges[idRamRange] == NULL)
1046	#endif
1047	{
1048	if (pVM->pgm.s.apRamRanges[idRamRange] == NIL_RTR3PTR)
1049	{
1050	/*
1051	* Commit it.
1052	*/
1053	#ifdef IN_RING0
1054	pVM->pgmr0.s.apRamRanges[idRamRange] = pRamRange;
1055	pVM->pgmr0.s.acRamRangePages[idRamRange] = cPages;
1056	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = hMemObj;
1057	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = hMapObj;
1058	pVM->pgmr0.s.idRamRangeMax = idRamRange;
1059	#endif
1060
1061	pVM->pgm.s.idRamRangeMax = idRamRange;
1062	#ifdef IN_RING0
1063	pVM->pgm.s.apRamRanges[idRamRange] = RTR0MemObjAddressR3(hMapObj);
1064	#else
1065	pVM->pgm.s.apRamRanges[idRamRange] = pRamRange;
1066	#endif
1067
1068	pRamRange->idRange = idRamRange;
1069	*pidNewRange = idRamRange;
1070
1071	PGM_UNLOCK(pVM);
1072	return VINF_SUCCESS;
1073	}
1074	}
1075
1076	/*
1077	* Bail out.
1078	*/
1079	rc = VERR_INTERNAL_ERROR_5;
1080	}
1081	else
1082	rc = VERR_PGM_TOO_MANY_RAM_RANGES;
1083	PGM_UNLOCK(pVM);
1084	}
1085	#ifdef IN_RING0
1086	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1087	#endif
1088	}
1089	#ifdef IN_RING0
1090	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1091	#else
1092	SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC);
1093	#endif
1094	}
1095	*pidNewRange = UINT32_MAX;
1096	return rc;
1097	}
1098
1099
1100	#ifdef IN_RING0
1101	/**
1102	* This is called during VM initialization to allocate a RAM range.
1103	*
1104	* The range is not entered into the lookup table, that is something the caller
1105	* has to do. The PGMPAGE entries are zero'ed, but otherwise uninitialized.
1106	*
1107	* @returns VBox status code.
1108	* @param pGVM Pointer to the global VM structure.
1109	* @param pReq Where to get the parameters and return the range ID.
1110	* @thread EMT(0)
1111	*/
1112	VMMR0_INT_DECL(int) PGMR0PhysAllocateRamRangeReq(PGVM pGVM, PPGMPHYSALLOCATERAMRANGEREQ pReq)
1113	{
1114	/*
1115	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1116	* while we're here).
1117	*/
1118	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1119	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1120
1121	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1122
1123	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1124	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_RAM_RANGE, VERR_OUT_OF_RANGE);
1125
1126	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGM_RAM_RANGE_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1127	VERR_INVALID_FLAGS);
1128
1129	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1130	VMSTATE const enmState = pGVM->enmVMState;
1131	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1132	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1133
1134	/*
1135	* Call common worker.
1136	*/
1137	return pgmPhysRamRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->fFlags, &pReq->idNewRange);
1138	}
1139	#endif /* IN_RING0 */
1140
1141
1142	/**
1143	* Frees a RAM range.
1144	*
1145	* This is not a typical occurence. Currently only used for a special MMIO2
1146	* saved state compatibility scenario involving PCNet and state saved before
1147	* VBox v4.3.6.
1148	*/
1149	static int pgmPhysRamRangeFree(PVMCC pVM, PPGMRAMRANGE pRamRange)
1150	{
1151	/*
1152	* Some basic input validation.
1153	*/
1154	AssertPtrReturn(pRamRange, VERR_INVALID_PARAMETER);
1155	uint32_t const idRamRange = ASMAtomicReadU32(&pRamRange->idRange);
1156	ASMCompilerBarrier();
1157	AssertReturn(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges), VERR_INVALID_PARAMETER);
1158	AssertReturn(pRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange], VERR_INVALID_PARAMETER);
1159	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_RESOURCE_BUSY);
1160
1161	/*
1162	* Kill the range pointers and associated data.
1163	*/
1164	pVM->pgm.s.apRamRanges[idRamRange] = NIL_RTR3PTR;
1165	#ifdef IN_RING0
1166	pVM->pgmr0.s.apRamRanges[idRamRange] = NULL;
1167	#endif
1168
1169	/*
1170	* Zap the pages and other RAM ranges properties to ensure there aren't any
1171	* stale references to anything hanging around should the freeing go awry.
1172	*/
1173	#ifdef IN_RING0
1174	uint32_t const cPages = pVM->pgmr0.s.acRamRangePages[idRamRange];
1175	pVM->pgmr0.s.acRamRangePages[idRamRange] = 0;
1176	#else
1177	uint32_t const cPages = pRamRange->cb >> GUEST_PAGE_SHIFT;
1178	#endif
1179	RT_BZERO(pRamRange->aPages, cPages * sizeof(pRamRange->aPages[0]));
1180
1181	pRamRange->fFlags = UINT32_MAX;
1182	pRamRange->cb = NIL_RTGCPHYS;
1183	pRamRange->pbR3 = NIL_RTR3PTR;
1184	pRamRange->pszDesc = NIL_RTR3PTR;
1185	pRamRange->paLSPages = NIL_RTR3PTR;
1186	pRamRange->idRange = UINT32_MAX / 8;
1187
1188	/*
1189	* Free the RAM range itself.
1190	*/
1191	#ifdef IN_RING0
1192	Assert(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] != NIL_RTR0MEMOBJ);
1193	int rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange], true /fFreeMappings/);
1194	if (RT_SUCCESS(rc))
1195	{
1196	pVM->pgmr0.s.ahRamRangeMapObjs[idRamRange] = NIL_RTR0MEMOBJ;
1197	rc = RTR0MemObjFree(pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange], true /fFreeMappings/);
1198	if (RT_SUCCESS(rc))
1199	pVM->pgmr0.s.ahRamRangeMemObjs[idRamRange] = NIL_RTR0MEMOBJ;
1200	}
1201	#else
1202	size_t const cbRamRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMRAMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1203	int rc = SUPR3PageFree(pRamRange, cbRamRange >> HOST_PAGE_SHIFT_DYNAMIC);
1204	#endif
1205
1206	/*
1207	* Decrease the max ID if removal was successful and this was the final
1208	* RAM range entry.
1209	*/
1210	if ( RT_SUCCESS(rc)
1211	&& idRamRange == pVM->CTX_EXPR(pgm, pgmr0, pgm).s.idRamRangeMax)
1212	{
1213	pVM->pgm.s.idRamRangeMax = idRamRange - 1;
1214	#ifdef IN_RING0
1215	pVM->pgmr0.s.idRamRangeMax = idRamRange - 1;
1216	#endif
1217	}
1218
1219	/*
1220	* Make sure the RAM range TLB does not contain any stale pointers to this range.
1221	*/
1222	pgmPhysInvalidRamRangeTlbs(pVM);
1223	return rc;
1224	}
1225
1226
1227
1228	/*********************************************************************************************************************************
1229	* MMIO2 *
1230	*********************************************************************************************************************************/
1231
1232	/**
1233	* Calculates the number of chunks
1234	*
1235	* @returns Number of registration chunk needed.
1236	* @param cb The size of the MMIO/MMIO2 range.
1237	* @param pcPagesPerChunk Where to return the number of guest pages tracked by
1238	* each chunk. Optional.
1239	*/
1240	DECLHIDDEN(uint16_t) pgmPhysMmio2CalcChunkCount(RTGCPHYS cb, uint32_t *pcPagesPerChunk)
1241	{
1242	/*
1243	* This is the same calculation as PGMR3PhysRegisterRam does, except we'll be
1244	* needing a few bytes extra the PGMREGMMIO2RANGE structure.
1245	*
1246	* Note! In additions, we've got a 24 bit sub-page range for MMIO2 ranges, leaving
1247	* us with an absolute maximum of 16777215 pages per chunk (close to 64 GB).
1248	*/
1249	AssertCompile(PGM_MAX_PAGES_PER_RAM_RANGE < _16M);
1250	uint32_t const cPagesPerChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
1251
1252	if (pcPagesPerChunk)
1253	*pcPagesPerChunk = cPagesPerChunk;
1254
1255	/* Calc the number of chunks we need. */
1256	RTGCPHYS const cGuestPages = cb >> GUEST_PAGE_SHIFT;
1257	uint16_t cChunks = (uint16_t)((cGuestPages + cPagesPerChunk - 1) / cPagesPerChunk);
1258	#ifdef IN_RING3
1259	AssertRelease((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages);
1260	#else
1261	AssertReturn((RTGCPHYS)cChunks * cPagesPerChunk >= cGuestPages, 0);
1262	#endif
1263	return cChunks;
1264	}
1265
1266
1267	/**
1268	* Worker for PGMR3PhysMmio2Register and PGMR0PhysMmio2RegisterReq.
1269	*
1270	* (The caller already know which MMIO2 region ID will be assigned and how many
1271	* chunks will be used, so no output parameters required.)
1272	*/
1273	DECLHIDDEN(int) pgmPhysMmio2RegisterWorker(PVMCC pVM, uint32_t const cGuestPages, uint8_t const idMmio2,
1274	const uint8_t cChunks, PPDMDEVINSR3 const pDevIns, uint8_t
1275	const iSubDev, uint8_t const iRegion, uint32_t const fFlags)
1276	{
1277	/*
1278	* Get the number of pages per chunk.
1279	*/
1280	uint32_t cGuestPagesPerChunk;
1281	AssertReturn(pgmPhysMmio2CalcChunkCount((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, &cGuestPagesPerChunk) == cChunks,
1282	VERR_PGM_PHYS_MMIO_EX_IPE);
1283	Assert(idMmio2 != 0);
1284
1285	/*
1286	* The first thing we need to do is the allocate the memory that will be
1287	* backing the whole range.
1288	*/
1289	RTGCPHYS const cbMmio2Backing = (RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT;
1290	uint32_t const cHostPages = (cbMmio2Backing + HOST_PAGE_SIZE_DYNAMIC - 1U) >> HOST_PAGE_SHIFT_DYNAMIC;
1291	size_t const cbMmio2Aligned = cHostPages << HOST_PAGE_SHIFT_DYNAMIC;
1292	R3PTRTYPE(uint8_t *) pbMmio2BackingR3 = NIL_RTR3PTR;
1293	#ifdef IN_RING0
1294	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1295	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1296	int rc = RTR0MemObjAllocPage(&hMemObj, cbMmio2Aligned, false /fExecutable/);
1297	# else
1298	int rc = RTR0MemObjAllocPhysNC(&hMemObj, cbMmio2Aligned, NIL_RTHCPHYS);
1299	# endif
1300	#else /* !IN_RING0 */
1301	AssertReturn(PGM_IS_IN_NEM_MODE(pVM), VERR_INTERNAL_ERROR_4);
1302	int rc = SUPR3PageAlloc(cHostPages, pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, (void **)&pbMmio2BackingR3);
1303	#endif /* !IN_RING0 */
1304	if (RT_SUCCESS(rc))
1305	{
1306	/*
1307	* Make sure it's is initialized to zeros before it's mapped to userland.
1308	*/
1309	#ifdef IN_RING0
1310	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1311	uint8_t pbMmio2BackingR0 = (uint8_t )RTR0MemObjAddress(hMemObj);
1312	AssertPtr(pbMmio2BackingR0);
1313	# endif
1314	rc = RTR0MemObjZeroInitialize(hMemObj, false /fForce/);
1315	AssertRCReturnStmt(rc, RTR0MemObjFree(hMemObj, true /fFreeMappings/), rc);
1316	#else
1317	RT_BZERO(pbMmio2BackingR3, cbMmio2Aligned);
1318	#endif
1319
1320	#ifdef IN_RING0
1321	/*
1322	* Map it into ring-3.
1323	*/
1324	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1325	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0, RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1326	if (RT_SUCCESS(rc))
1327	{
1328	pbMmio2BackingR3 = RTR0MemObjAddressR3(hMapObj);
1329	#endif
1330
1331	/*
1332	* Create the MMIO2 registration records and associated RAM ranges.
1333	* The RAM range allocation may fail here.
1334	*/
1335	RTGCPHYS offMmio2Backing = 0;
1336	uint32_t cGuestPagesLeft = cGuestPages;
1337	for (uint32_t iChunk = 0, idx = idMmio2 - 1; iChunk < cChunks; iChunk++, idx++)
1338	{
1339	uint32_t const cPagesTrackedByChunk = RT_MIN(cGuestPagesLeft, cGuestPagesPerChunk);
1340
1341	/*
1342	* Allocate the RAM range for this chunk.
1343	*/
1344	uint32_t idRamRange = UINT32_MAX;
1345	rc = pgmPhysRamRangeAllocCommon(pVM, cPagesTrackedByChunk, PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, &idRamRange);
1346	if (RT_FAILURE(rc))
1347	{
1348	/* We only zap the pointers to the backing storage.
1349	PGMR3Term and friends will clean up the RAM ranges and stuff. */
1350	while (iChunk-- > 0)
1351	{
1352	idx--;
1353	#ifdef IN_RING0
1354	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1355	# ifndef VBOX_WITH_LINEAR_HOST_PHYS_MEM
1356	pVM->pgmr0.s.apbMmio2Backing[idx] = NULL;
1357	# endif
1358	#endif
1359
1360	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1361	pMmio2->pbR3 = NIL_RTR3PTR;
1362
1363	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1364	pRamRange->pbR3 = NIL_RTR3PTR;
1365	RT_BZERO(&pRamRange->aPages[0], sizeof(pRamRange->aPages) * cGuestPagesPerChunk);
1366	}
1367	break;
1368	}
1369
1370	pVM->pgm.s.apMmio2RamRanges[idx] = pVM->pgm.s.apRamRanges[idRamRange];
1371	#ifdef IN_RING0
1372	pVM->pgmr0.s.apMmio2RamRanges[idx] = pVM->pgmr0.s.apRamRanges[idRamRange];
1373	pVM->pgmr0.s.acMmio2RangePages[idx] = cPagesTrackedByChunk;
1374	#endif
1375
1376	/* Initialize the RAM range. */
1377	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
1378	pRamRange->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1379	uint32_t iDstPage = cPagesTrackedByChunk;
1380	#ifdef IN_RING0
1381	AssertRelease(HOST_PAGE_SHIFT == GUEST_PAGE_SHIFT);
1382	while (iDstPage-- > 0)
1383	{
1384	RTHCPHYS HCPhys = RTR0MemObjGetPagePhysAddr(hMemObj, iDstPage + (offMmio2Backing >> HOST_PAGE_SHIFT));
1385	Assert(HCPhys != NIL_RTHCPHYS);
1386	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], HCPhys, PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1387	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1388	}
1389	#else
1390	Assert(PGM_IS_IN_NEM_MODE(pVM));
1391	while (iDstPage-- > 0)
1392	PGM_PAGE_INIT(&pRamRange->aPages[iDstPage], UINT64_C(0x0000ffffffff0000),
1393	PGM_MMIO2_PAGEID_MAKE(idMmio2, iDstPage),
1394	PGMPAGETYPE_MMIO2, PGM_PAGE_STATE_ALLOCATED);
1395	#endif
1396
1397	/*
1398	* Initialize the MMIO2 registration structure.
1399	*/
1400	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
1401	pMmio2->pDevInsR3 = pDevIns;
1402	pMmio2->pbR3 = pbMmio2BackingR3 + offMmio2Backing;
1403	pMmio2->fFlags = 0;
1404	if (iChunk == 0)
1405	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_FIRST_CHUNK;
1406	if (iChunk + 1 == cChunks)
1407	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_LAST_CHUNK;
1408	if (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
1409	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES;
1410
1411	pMmio2->iSubDev = iSubDev;
1412	pMmio2->iRegion = iRegion;
1413	pMmio2->idSavedState = UINT8_MAX;
1414	pMmio2->idMmio2 = idMmio2 + iChunk;
1415	pMmio2->idRamRange = idRamRange;
1416	Assert(pMmio2->idRamRange == idRamRange);
1417	pMmio2->GCPhys = NIL_RTGCPHYS;
1418	pMmio2->cbReal = (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1419	pMmio2->pPhysHandlerR3 = NIL_RTR3PTR; /* Pre-alloc is done by ring-3 caller. */
1420	pMmio2->paLSPages = NIL_RTR3PTR;
1421
1422	#if defined(IN_RING0) && !defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
1423	pVM->pgmr0.s.apbMmio2Backing[idx] = &pbMmio2BackingR0[offMmio2Backing];
1424	#endif
1425
1426	/* Advance */
1427	cGuestPagesLeft -= cPagesTrackedByChunk;
1428	offMmio2Backing += (RTGCPHYS)cPagesTrackedByChunk << GUEST_PAGE_SHIFT;
1429	} /* chunk alloc loop */
1430	Assert(cGuestPagesLeft == 0 \|\| RT_FAILURE_NP(rc));
1431	if (RT_SUCCESS(rc))
1432	{
1433	/*
1434	* Account for pages and ring-0 memory objects.
1435	*/
1436	pVM->pgm.s.cAllPages += cGuestPages;
1437	pVM->pgm.s.cPrivatePages += cGuestPages;
1438	#ifdef IN_RING0
1439	pVM->pgmr0.s.ahMmio2MemObjs[idMmio2 - 1] = hMemObj;
1440	pVM->pgmr0.s.ahMmio2MapObjs[idMmio2 - 1] = hMapObj;
1441	#endif
1442	pVM->pgm.s.cMmio2Ranges = idMmio2 + cChunks - 1U;
1443
1444	/*
1445	* Done!.
1446	*/
1447	return VINF_SUCCESS;
1448	}
1449
1450	/*
1451	* Bail.
1452	*/
1453	#ifdef IN_RING0
1454	RTR0MemObjFree(hMapObj, true /fFreeMappings/);
1455	}
1456	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1457	#else
1458	SUPR3PageFree(pbMmio2BackingR3, cHostPages);
1459	#endif
1460	}
1461	else
1462	LogRel(("pgmPhysMmio2RegisterWorker: Failed to allocate %RGp bytes of MMIO2 backing memory: %Rrc\n", cbMmio2Aligned, rc));
1463	return rc;
1464	}
1465
1466
1467	#ifdef IN_RING0
1468	/**
1469	* This is called during VM initialization to create an MMIO2 range.
1470	*
1471	* This does everything except setting the PGMRAMRANGE::pszDesc to a non-zero
1472	* value and preallocating the access handler for dirty bitmap tracking.
1473	*
1474	* The caller already knows which MMIO2 ID will be assigned to the registration
1475	* and how many chunks it requires, so there are no output fields in the request
1476	* structure.
1477	*
1478	* @returns VBox status code.
1479	* @param pGVM Pointer to the global VM structure.
1480	* @param pReq Where to get the parameters.
1481	* @thread EMT(0)
1482	*/
1483	VMMR0_INT_DECL(int) PGMR0PhysMmio2RegisterReq(PGVM pGVM, PPGMPHYSMMIO2REGISTERREQ pReq)
1484	{
1485	/*
1486	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1487	* while we're here).
1488	*/
1489	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1490	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1491
1492	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1493	VMSTATE const enmState = pGVM->enmVMState;
1494	AssertMsgReturn( enmState == VMSTATE_CREATING
1495	\|\| enmState == VMSTATE_LOADING /* pre 4.3.6 state loading needs to ignore a MMIO2 region in PCNet. */
1496	, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1497	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1498
1499	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1500	AssertReturn(GUEST_PAGE_SIZE == HOST_PAGE_SIZE_DYNAMIC, VERR_INCOMPATIBLE_CONFIG);
1501
1502	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1503	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_MMIO2_REGION, VERR_OUT_OF_RANGE);
1504	AssertReturn(pReq->cGuestPages <= (MM_MMIO_64_MAX >> GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
1505
1506	AssertMsgReturn(!(pReq->fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), ("fFlags=%#x\n", pReq->fFlags), VERR_INVALID_FLAGS);
1507
1508	AssertMsgReturn( pReq->cChunks > 0
1509	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES
1510	&& pReq->cChunks == pgmPhysMmio2CalcChunkCount((RTGCPHYS)pReq->cGuestPages << GUEST_PAGE_SHIFT, NULL),
1511	("cChunks=%#x cGuestPages=%#x\n", pReq->cChunks, pReq->cGuestPages),
1512	VERR_INVALID_PARAMETER);
1513
1514	AssertMsgReturn( pReq->idMmio2 != 0
1515	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1516	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1517	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1518	VERR_INVALID_PARAMETER);
1519
1520	for (uint32_t iChunk = 0, idx = pReq->idMmio2 - 1; iChunk < pReq->cChunks; iChunk++, idx++)
1521	{
1522	AssertReturn(pGVM->pgmr0.s.ahMmio2MapObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1523	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1524	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL, VERR_INVALID_STATE);
1525	}
1526
1527	/*
1528	* Make sure we're owning the PGM lock (caller should be), recheck idMmio2
1529	* and call the worker function we share with ring-3.
1530	*/
1531	int rc = PGM_LOCK(pGVM);
1532	AssertRCReturn(rc, rc);
1533
1534	AssertReturnStmt(pGVM->pgm.s.cMmio2Ranges + 1U == pReq->idMmio2,
1535	PGM_UNLOCK(pGVM), VERR_INVALID_PARAMETER);
1536	AssertReturnStmt(pGVM->pgmr0.s.idRamRangeMax + 1U + pReq->cChunks <= RT_ELEMENTS(pGVM->pgmr0.s.apRamRanges),
1537	PGM_UNLOCK(pGVM), VERR_PGM_TOO_MANY_RAM_RANGES);
1538
1539	rc = pgmPhysMmio2RegisterWorker(pGVM, pReq->cGuestPages, pReq->idMmio2, pReq->cChunks,
1540	pReq->pDevIns, pReq->iSubDev, pReq->iRegion, pReq->fFlags);
1541
1542	PGM_UNLOCK(pGVM);
1543	return rc;
1544	}
1545	#endif /* IN_RING0 */
1546
1547
1548
1549	/**
1550	* Worker for PGMR3PhysMmio2Deregister & PGMR0PhysMmio2DeregisterReq.
1551	*/
1552	DECLHIDDEN(int) pgmPhysMmio2DeregisterWorker(PVMCC pVM, uint8_t idMmio2, uint8_t cChunks, PPDMDEVINSR3 pDevIns)
1553	{
1554	/*
1555	* The caller shall have made sure all this is true, but we check again
1556	* since we're paranoid.
1557	*/
1558	AssertReturn(idMmio2 > 0 && idMmio2 <= RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), VERR_INTERNAL_ERROR_2);
1559	AssertReturn(cChunks >= 1, VERR_INTERNAL_ERROR_2);
1560	uint8_t const idxFirst = idMmio2 - 1U;
1561	AssertReturn(idxFirst + cChunks <= pVM->pgm.s.cMmio2Ranges, VERR_INTERNAL_ERROR_2);
1562	uint32_t cGuestPages = 0; /* (For accounting and calulating backing memory size) */
1563	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
1564	{
1565	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 == pDevIns, VERR_NOT_OWNER);
1566	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED), VERR_RESOURCE_BUSY);
1567	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1568	if (iChunk == 0)
1569	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK, VERR_INVALID_PARAMETER);
1570	else
1571	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_INVALID_PARAMETER);
1572	if (iChunk + 1 == cChunks)
1573	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK, VERR_INVALID_PARAMETER);
1574	else
1575	AssertReturn(!(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK), VERR_INVALID_PARAMETER);
1576	AssertReturn(pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 == NIL_RTR3PTR, VERR_INVALID_STATE); /* caller shall free this */
1577
1578	#ifdef IN_RING0
1579	cGuestPages += pVM->pgmr0.s.acMmio2RangePages[idx];
1580	#else
1581	cGuestPages += pVM->pgm.s.aMmio2Ranges[idx].cbReal >> GUEST_PAGE_SHIFT;
1582	#endif
1583
1584	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1585	AssertPtrReturn(pRamRange, VERR_INVALID_STATE);
1586	AssertReturn(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO_EX, VERR_INVALID_STATE);
1587	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_INVALID_STATE);
1588	AssertReturn(pRamRange->GCPhysLast == NIL_RTGCPHYS, VERR_INVALID_STATE);
1589	}
1590
1591	/*
1592	* Remove everything except the backing memory first. We work the ranges
1593	* in reverse so that we can reduce the max RAM range ID when possible.
1594	*/
1595	#ifdef IN_RING3
1596	uint8_t * const pbMmio2Backing = pVM->pgm.s.aMmio2Ranges[idxFirst].pbR3;
1597	RTGCPHYS const cbMmio2Backing = RT_ALIGN_T((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT, HOST_PAGE_SIZE_DYNAMIC, RTGCPHYS);
1598	#endif
1599
1600	int rc = VINF_SUCCESS;
1601	uint32_t iChunk = cChunks;
1602	while (iChunk-- > 0)
1603	{
1604	uint32_t const idx = idxFirst + iChunk;
1605	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apMmio2RamRanges[idx];
1606
1607	/* Zap the MMIO2 region data. */
1608	pVM->pgm.s.apMmio2RamRanges[idx] = NIL_RTR3PTR;
1609	#ifdef IN_RING0
1610	pVM->pgmr0.s.apMmio2RamRanges[idx] = NULL;
1611	pVM->pgmr0.s.acMmio2RangePages[idx] = 0;
1612	#endif
1613	pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 = NIL_RTR3PTR;
1614	pVM->pgm.s.aMmio2Ranges[idx].pbR3 = NIL_RTR3PTR;
1615	pVM->pgm.s.aMmio2Ranges[idx].fFlags = 0;
1616	pVM->pgm.s.aMmio2Ranges[idx].iSubDev = UINT8_MAX;
1617	pVM->pgm.s.aMmio2Ranges[idx].iRegion = UINT8_MAX;
1618	pVM->pgm.s.aMmio2Ranges[idx].idSavedState = UINT8_MAX;
1619	pVM->pgm.s.aMmio2Ranges[idx].idMmio2 = UINT8_MAX;
1620	pVM->pgm.s.aMmio2Ranges[idx].idRamRange = UINT16_MAX;
1621	pVM->pgm.s.aMmio2Ranges[idx].GCPhys = NIL_RTGCPHYS;
1622	pVM->pgm.s.aMmio2Ranges[idx].cbReal = 0;
1623	pVM->pgm.s.aMmio2Ranges[idx].pPhysHandlerR3 = NIL_RTR3PTR;
1624	pVM->pgm.s.aMmio2Ranges[idx].paLSPages = NIL_RTR3PTR;
1625
1626	/* Free the RAM range. */
1627	int rc2 = pgmPhysRamRangeFree(pVM, pRamRange);
1628	AssertLogRelMsgStmt(RT_SUCCESS(rc2), ("rc=%Rrc idx=%u chunk=%u/%u\n", rc, idx, iChunk + 1, cChunks),
1629	rc = RT_SUCCESS(rc) ? rc2 : rc);
1630	}
1631
1632	/*
1633	* Final removal frees up the backing memory.
1634	*/
1635	#ifdef IN_RING3
1636	int const rcBacking = SUPR3PageFree(pbMmio2Backing, cbMmio2Backing >> HOST_PAGE_SHIFT_DYNAMIC);
1637	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking), ("rc=%Rrc %p LB %#zx\n", rcBacking, pbMmio2Backing, cbMmio2Backing),
1638	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1639	#else
1640	int rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], true /fFreeMappings/);
1641	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1642	("rc=%Rrc ahMmio2MapObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MapObjs[idxFirst], idxFirst),
1643	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1644	if (RT_SUCCESS(rcBacking))
1645	{
1646	pVM->pgmr0.s.ahMmio2MapObjs[idxFirst] = NIL_RTR0MEMOBJ;
1647
1648	rcBacking = RTR0MemObjFree(pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], true /fFreeMappings/);
1649	AssertLogRelMsgStmt(RT_SUCCESS(rcBacking),
1650	("rc=%Rrc ahMmio2MemObjs[%u]=%p\n", rcBacking, pVM->pgmr0.s.ahMmio2MemObjs[idxFirst], idxFirst),
1651	rc = RT_SUCCESS(rc) ? rcBacking : rc);
1652	if (RT_SUCCESS(rcBacking))
1653	pVM->pgmr0.s.ahMmio2MemObjs[idxFirst] = NIL_RTR0MEMOBJ;
1654	}
1655	#endif
1656
1657	/*
1658	* Decrease the MMIO2 count if these were the last ones.
1659	*/
1660	if (idxFirst + cChunks == pVM->pgm.s.cMmio2Ranges)
1661	pVM->pgm.s.cMmio2Ranges = idxFirst;
1662
1663	/*
1664	* Update page count stats.
1665	*/
1666	pVM->pgm.s.cAllPages -= cGuestPages;
1667	pVM->pgm.s.cPrivatePages -= cGuestPages;
1668
1669	return rc;
1670	}
1671
1672
1673	#ifdef IN_RING0
1674	/**
1675	* This is called during VM state loading to deregister an obsolete MMIO2 range.
1676	*
1677	* This does everything except TLB flushing and releasing the access handler.
1678	* The ranges must be unmapped and wihtout preallocated access handlers.
1679	*
1680	* @returns VBox status code.
1681	* @param pGVM Pointer to the global VM structure.
1682	* @param pReq Where to get the parameters.
1683	* @thread EMT(0)
1684	*/
1685	VMMR0_INT_DECL(int) PGMR0PhysMmio2DeregisterReq(PGVM pGVM, PPGMPHYSMMIO2DEREGISTERREQ pReq)
1686	{
1687	/*
1688	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1689	* while we're here).
1690	*/
1691	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1692	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1693
1694	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1695	/* Only LOADING, as this is special purpose for removing an unwanted PCNet MMIO2 region. */
1696	VMSTATE const enmState = pGVM->enmVMState;
1697	AssertMsgReturn(enmState == VMSTATE_LOADING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1698	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1699
1700	AssertMsgReturn( pReq->cChunks > 0
1701	&& pReq->cChunks < PGM_MAX_MMIO2_RANGES,
1702	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1703	VERR_INVALID_PARAMETER);
1704
1705	AssertMsgReturn( pReq->idMmio2 != 0
1706	&& pReq->idMmio2 <= PGM_MAX_MMIO2_RANGES
1707	&& (unsigned)pReq->idMmio2 + pReq->cChunks - 1U <= PGM_MAX_MMIO2_RANGES,
1708	("idMmio2=%#x cChunks=%#x\n", pReq->idMmio2, pReq->cChunks),
1709	VERR_INVALID_PARAMETER);
1710
1711	/*
1712	* Validate that the requested range is for exactly one MMIO2 registration.
1713	*
1714	* This is safe to do w/o the lock because registration and deregistration
1715	* is restricted to EMT0, and we're on EMT0 so can't race ourselves.
1716	*/
1717
1718	/* Check that the first entry is valid and has a memory object for the backing memory. */
1719	uint32_t idx = pReq->idMmio2 - 1;
1720	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1721	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1722
1723	/* Any additional regions must also have RAM ranges, but shall not have any backing memory. */
1724	idx++;
1725	for (uint32_t iChunk = 1; iChunk < pReq->cChunks; iChunk++, idx++)
1726	{
1727	AssertReturn(pGVM->pgmr0.s.apMmio2RamRanges[idx] != NULL, VERR_INVALID_STATE);
1728	AssertReturn(pGVM->pgmr0.s.ahMmio2MemObjs[idx] == NIL_RTR0MEMOBJ, VERR_INVALID_STATE);
1729	}
1730
1731	/* Check that the next entry is for a different region. */
1732	AssertReturn( idx >= RT_ELEMENTS(pGVM->pgmr0.s.apMmio2RamRanges)
1733	\|\| pGVM->pgmr0.s.apMmio2RamRanges[idx] == NULL
1734	\|\| pGVM->pgmr0.s.ahMmio2MemObjs[idx] != NIL_RTR0MEMOBJ,
1735	VERR_INVALID_PARAMETER);
1736
1737	/*
1738	* Make sure we're owning the PGM lock (caller should be) and call the
1739	* common worker code.
1740	*/
1741	int rc = PGM_LOCK(pGVM);
1742	AssertRCReturn(rc, rc);
1743
1744	rc = pgmPhysMmio2DeregisterWorker(pGVM, pReq->idMmio2, pReq->cChunks, pReq->pDevIns);
1745
1746	PGM_UNLOCK(pGVM);
1747	return rc;
1748	}
1749	#endif /* IN_RING0 */
1750
1751
1752
1753
1754	/*********************************************************************************************************************************
1755	* ROM *
1756	*********************************************************************************************************************************/
1757
1758
1759	/**
1760	* Common worker for pgmR3PhysRomRegisterLocked and
1761	* PGMR0PhysRomAllocateRangeReq.
1762	*/
1763	DECLHIDDEN(int) pgmPhysRomRangeAllocCommon(PVMCC pVM, uint32_t cPages, uint8_t idRomRange, uint32_t fFlags)
1764	{
1765	/*
1766	* Allocate the ROM range structure and map it into ring-3.
1767	*/
1768	size_t const cbRomRange = RT_ALIGN_Z(RT_UOFFSETOF_DYN(PGMROMRANGE, aPages[cPages]), HOST_PAGE_SIZE_DYNAMIC);
1769	#ifdef IN_RING0
1770	RTR0MEMOBJ hMemObj = NIL_RTR0MEMOBJ;
1771	int rc = RTR0MemObjAllocPage(&hMemObj, cbRomRange, false /fExecutable/);
1772	#else
1773	PPGMROMRANGE pRomRange;
1774	int rc = SUPR3PageAlloc(cbRomRange >> HOST_PAGE_SHIFT_DYNAMIC, 0 /fFlags/, (void **)&pRomRange);
1775	#endif
1776	if (RT_SUCCESS(rc))
1777	{
1778	/* Zero the memory and do basic range init before mapping it into userland. */
1779	#ifdef IN_RING0
1780	PPGMROMRANGE const pRomRange = (PPGMROMRANGE)RTR0MemObjAddress(hMemObj);
1781	if (!RTR0MemObjWasZeroInitialized(hMemObj))
1782	#endif
1783	RT_BZERO(pRomRange, cbRomRange);
1784
1785	pRomRange->GCPhys = NIL_RTGCPHYS;
1786	pRomRange->GCPhysLast = NIL_RTGCPHYS;
1787	pRomRange->cb = (RTGCPHYS)cPages << GUEST_PAGE_SHIFT;
1788	pRomRange->fFlags = fFlags;
1789	pRomRange->idSavedState = UINT8_MAX;
1790	pRomRange->idRamRange = UINT16_MAX;
1791	pRomRange->cbOriginal = 0;
1792	pRomRange->pvOriginal = NIL_RTR3PTR;
1793	pRomRange->pszDesc = NIL_RTR3PTR;
1794
1795	#ifdef IN_RING0
1796	/* Map it into userland. */
1797	RTR0MEMOBJ hMapObj = NIL_RTR0MEMOBJ;
1798	rc = RTR0MemObjMapUser(&hMapObj, hMemObj, (RTR3PTR)-1, 0 /uAlignment/,
1799	RTMEM_PROT_READ \| RTMEM_PROT_WRITE, NIL_RTR0PROCESS);
1800	if (RT_SUCCESS(rc))
1801	#endif
1802	{
1803	/*
1804	* Grab the lock (unlikely to fail or block as caller typically owns it already).
1805	*/
1806	rc = PGM_LOCK(pVM);
1807	if (RT_SUCCESS(rc))
1808	{
1809	/*
1810	* Check that idRomRange is still free.
1811	*/
1812	if (idRomRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRomRanges))
1813	{
1814	#ifdef IN_RING0
1815	if (pVM->pgmr0.s.apRomRanges[idRomRange] == NULL)
1816	#endif
1817	{
1818	if ( pVM->pgm.s.apRomRanges[idRomRange] == NIL_RTR3PTR
1819	&& pVM->pgm.s.cRomRanges == idRomRange)
1820	{
1821	/*
1822	* Commit it.
1823	*/
1824	#ifdef IN_RING0
1825	pVM->pgmr0.s.apRomRanges[idRomRange] = pRomRange;
1826	pVM->pgmr0.s.acRomRangePages[idRomRange] = cPages;
1827	pVM->pgmr0.s.ahRomRangeMemObjs[idRomRange] = hMemObj;
1828	pVM->pgmr0.s.ahRomRangeMapObjs[idRomRange] = hMapObj;
1829	#endif
1830
1831	pVM->pgm.s.cRomRanges = idRomRange + 1;
1832	#ifdef IN_RING0
1833	pVM->pgm.s.apRomRanges[idRomRange] = RTR0MemObjAddressR3(hMapObj);
1834	#else
1835	pVM->pgm.s.apRomRanges[idRomRange] = pRomRange;
1836	#endif
1837
1838	PGM_UNLOCK(pVM);
1839	return VINF_SUCCESS;
1840	}
1841	}
1842
1843	/*
1844	* Bail out.
1845	*/
1846	rc = VERR_INTERNAL_ERROR_5;
1847	}
1848	else
1849	rc = VERR_PGM_TOO_MANY_ROM_RANGES;
1850	PGM_UNLOCK(pVM);
1851	}
1852	#ifdef IN_RING0
1853	RTR0MemObjFree(hMapObj, false /fFreeMappings/);
1854	#endif
1855	}
1856	#ifdef IN_RING0
1857	RTR0MemObjFree(hMemObj, true /fFreeMappings/);
1858	#else
1859	SUPR3PageFree(pRomRange, cbRomRange >> HOST_PAGE_SHIFT_DYNAMIC);
1860	#endif
1861	}
1862	return rc;
1863	}
1864
1865
1866	#ifdef IN_RING0
1867	/**
1868	* This is called during VM initialization to allocate a ROM range.
1869	*
1870	* The page array is zeroed, the rest is initialized as best we can based on the
1871	* information in @a pReq.
1872	*
1873	* @returns VBox status code.
1874	* @param pGVM Pointer to the global VM structure.
1875	* @param pReq Where to get the parameters and return the range ID.
1876	* @thread EMT(0)
1877	*/
1878	VMMR0_INT_DECL(int) PGMR0PhysRomAllocateRangeReq(PGVM pGVM, PPGMPHYSROMALLOCATERANGEREQ pReq)
1879	{
1880	/*
1881	* Validate input (ASSUME pReq is a copy and can't be modified by ring-3
1882	* while we're here).
1883	*/
1884	AssertPtrReturn(pReq, VERR_INVALID_POINTER);
1885	AssertMsgReturn(pReq->Hdr.cbReq == sizeof(pReq), ("%#x < %#zx\n", pReq->Hdr.cbReq, sizeof(pReq)), VERR_INVALID_PARAMETER);
1886
1887	AssertReturn(pReq->cbGuestPage == GUEST_PAGE_SIZE, VERR_INCOMPATIBLE_CONFIG);
1888
1889	AssertReturn(pReq->cGuestPages > 0, VERR_OUT_OF_RANGE);
1890	AssertReturn(pReq->cGuestPages <= PGM_MAX_PAGES_PER_ROM_RANGE, VERR_OUT_OF_RANGE);
1891
1892	AssertMsgReturn(!(pReq->fFlags & ~(uint32_t)PGMPHYS_ROM_FLAGS_VALID_MASK), ("fFlags=%#RX32\n", pReq->fFlags),
1893	VERR_INVALID_FLAGS);
1894
1895	AssertReturn(pReq->idRomRange < RT_ELEMENTS(pGVM->pgmr0.s.apRomRanges), VERR_OUT_OF_RANGE);
1896	AssertReturn(pReq->idRomRange == pGVM->pgm.s.cRomRanges, VERR_OUT_OF_RANGE);
1897
1898	/** @todo better VM state guard, enmVMState is ring-3 writable. */
1899	VMSTATE const enmState = pGVM->enmVMState;
1900	AssertMsgReturn(enmState == VMSTATE_CREATING, ("enmState=%d\n", enmState), VERR_VM_INVALID_VM_STATE);
1901	VM_ASSERT_EMT0_RETURN(pGVM, VERR_VM_THREAD_NOT_EMT);
1902
1903	/*
1904	* Call common worker.
1905	*/
1906	return pgmPhysRomRangeAllocCommon(pGVM, pReq->cGuestPages, pReq->idRomRange, pReq->fFlags);
1907	}
1908	#endif /* IN_RING0 */
1909
1910
1911	/*********************************************************************************************************************************
1912	* Other stuff
1913	*********************************************************************************************************************************/
1914
1915
1916
1917	#if defined(VBOX_VMM_TARGET_X86)
1918	/**
1919	* Checks if Address Gate 20 is enabled or not.
1920	*
1921	* @returns true if enabled.
1922	* @returns false if disabled.
1923	* @param pVCpu The cross context virtual CPU structure.
1924	*/
1925	VMMDECL(bool) PGMPhysIsA20Enabled(PVMCPU pVCpu)
1926	{
1927	/* Must check that pVCpu isn't NULL here because PDM device helper are a little lazy. */
1928	LogFlow(("PGMPhysIsA20Enabled %d\n", pVCpu && pVCpu->pgm.s.fA20Enabled));
1929	return pVCpu && pVCpu->pgm.s.fA20Enabled;
1930	}
1931	#endif
1932
1933
1934	/**
1935	* Validates a GC physical address.
1936	*
1937	* @returns true if valid.
1938	* @returns false if invalid.
1939	* @param pVM The cross context VM structure.
1940	* @param GCPhys The physical address to validate.
1941	*/
1942	VMMDECL(bool) PGMPhysIsGCPhysValid(PVMCC pVM, RTGCPHYS GCPhys)
1943	{
1944	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1945	return pPage != NULL;
1946	}
1947
1948
1949	/**
1950	* Checks if a GC physical address is a normal page,
1951	* i.e. not ROM, MMIO or reserved.
1952	*
1953	* @returns true if normal.
1954	* @returns false if invalid, ROM, MMIO or reserved page.
1955	* @param pVM The cross context VM structure.
1956	* @param GCPhys The physical address to check.
1957	*/
1958	VMMDECL(bool) PGMPhysIsGCPhysNormal(PVMCC pVM, RTGCPHYS GCPhys)
1959	{
1960	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
1961	return pPage
1962	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM;
1963	}
1964
1965
1966	/**
1967	* Converts a GC physical address to a HC physical address.
1968	*
1969	* @returns VINF_SUCCESS on success.
1970	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
1971	* page but has no physical backing.
1972	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
1973	* GC physical address.
1974	*
1975	* @param pVM The cross context VM structure.
1976	* @param GCPhys The GC physical address to convert.
1977	* @param pHCPhys Where to store the HC physical address on success.
1978	*/
1979	VMM_INT_DECL(int) PGMPhysGCPhys2HCPhys(PVMCC pVM, RTGCPHYS GCPhys, PRTHCPHYS pHCPhys)
1980	{
1981	PGM_LOCK_VOID(pVM);
1982	PPGMPAGE pPage;
1983	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
1984	if (RT_SUCCESS(rc))
1985	*pHCPhys = PGM_PAGE_GET_HCPHYS(pPage) \| (GCPhys & GUEST_PAGE_OFFSET_MASK);
1986	PGM_UNLOCK(pVM);
1987	return rc;
1988	}
1989
1990
1991	/**
1992	* Invalidates all page mapping TLBs.
1993	*
1994	* @param pVM The cross context VM structure.
1995	* @param fInRendezvous Set if we're in a rendezvous.
1996	*/
1997	void pgmPhysInvalidatePageMapTLB(PVMCC pVM, bool fInRendezvous)
1998	{
1999	PGM_LOCK_VOID(pVM);
2000	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushes);
2001
2002	/* Clear the R3 & R0 TLBs completely. */
2003	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR0.aEntries); i++)
2004	{
2005	pVM->pgm.s.PhysTlbR0.aEntries[i].GCPhys = NIL_RTGCPHYS;
2006	pVM->pgm.s.PhysTlbR0.aEntries[i].pPage = 0;
2007	pVM->pgm.s.PhysTlbR0.aEntries[i].pv = 0;
2008	}
2009
2010	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
2011	{
2012	pVM->pgm.s.PhysTlbR3.aEntries[i].GCPhys = NIL_RTGCPHYS;
2013	pVM->pgm.s.PhysTlbR3.aEntries[i].pPage = 0;
2014	pVM->pgm.s.PhysTlbR3.aEntries[i].pMap = 0;
2015	pVM->pgm.s.PhysTlbR3.aEntries[i].pv = 0;
2016	}
2017
2018	/* For the per VCPU lockless TLBs, we only invalid the GCPhys members so that
2019	anyone concurrently using the entry can safely continue to do so while any
2020	subsequent attempts to use it will fail. (Emulating a scenario where we
2021	lost the PGM lock race and the concurrent TLB user wont it.) */
2022	VMCC_FOR_EACH_VMCPU(pVM)
2023	{
2024	if (!fInRendezvous && pVCpu != VMMGetCpu(pVM))
2025	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2026	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2027	else
2028	for (unsigned idx = 0; idx < RT_ELEMENTS(pVCpu->pgm.s.PhysTlb.aEntries); idx++)
2029	pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2030	}
2031	VMCC_FOR_EACH_VMCPU_END(pVM);
2032
2033	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MISC);
2034	PGM_UNLOCK(pVM);
2035	}
2036
2037
2038	/**
2039	* Invalidates a page mapping TLB entry
2040	*
2041	* @param pVM The cross context VM structure.
2042	* @param GCPhys GCPhys entry to flush
2043	*
2044	* @note Caller is responsible for calling IEMTlbInvalidateAllPhysicalAllCpus
2045	* when needed.
2046	*/
2047	void pgmPhysInvalidatePageMapTLBEntry(PVMCC pVM, RTGCPHYS GCPhys)
2048	{
2049	PGM_LOCK_ASSERT_OWNER(pVM);
2050
2051	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatPageMapTlbFlushEntry);
2052
2053	unsigned const idx = PGM_PAGER3MAPTLB_IDX(GCPhys);
2054
2055	pVM->pgm.s.PhysTlbR0.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2056	pVM->pgm.s.PhysTlbR0.aEntries[idx].pPage = 0;
2057	pVM->pgm.s.PhysTlbR0.aEntries[idx].pv = 0;
2058
2059	pVM->pgm.s.PhysTlbR3.aEntries[idx].GCPhys = NIL_RTGCPHYS;
2060	pVM->pgm.s.PhysTlbR3.aEntries[idx].pPage = 0;
2061	pVM->pgm.s.PhysTlbR3.aEntries[idx].pMap = 0;
2062	pVM->pgm.s.PhysTlbR3.aEntries[idx].pv = 0;
2063
2064	/* For the per VCPU lockless TLBs, we only invalid the GCPhys member so that
2065	anyone concurrently using the entry can safely continue to do so while any
2066	subsequent attempts to use it will fail. (Emulating a scenario where we
2067	lost the PGM lock race and the concurrent TLB user wont it.) */
2068	VMCC_FOR_EACH_VMCPU(pVM)
2069	{
2070	ASMAtomicWriteU64(&pVCpu->pgm.s.PhysTlb.aEntries[idx].GCPhys, NIL_RTGCPHYS);
2071	}
2072	VMCC_FOR_EACH_VMCPU_END(pVM);
2073	}
2074
2075	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2076
2077	/**
2078	* Makes sure that there is at least one handy page ready for use.
2079	*
2080	* This will also take the appropriate actions when reaching water-marks.
2081	*
2082	* @returns VBox status code.
2083	* @retval VINF_SUCCESS on success.
2084	* @retval VERR_EM_NO_MEMORY if we're really out of memory.
2085	*
2086	* @param pVM The cross context VM structure.
2087	*
2088	* @remarks Must be called from within the PGM critical section. It may
2089	* nip back to ring-3/0 in some cases.
2090	*/
2091	static int pgmPhysEnsureHandyPage(PVMCC pVM)
2092	{
2093	AssertMsg(pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", pVM->pgm.s.cHandyPages));
2094
2095	/*
2096	* Do we need to do anything special?
2097	*/
2098	# ifdef IN_RING3
2099	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_R3_ALLOC))
2100	# else
2101	if (pVM->pgm.s.cHandyPages <= RT_MAX(PGM_HANDY_PAGES_SET_FF, PGM_HANDY_PAGES_RZ_TO_R3))
2102	# endif
2103	{
2104	/*
2105	* Allocate pages only if we're out of them, or in ring-3, almost out.
2106	*/
2107	# ifdef IN_RING3
2108	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_R3_ALLOC)
2109	# else
2110	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_ALLOC)
2111	# endif
2112	{
2113	Log(("PGM: cHandyPages=%u out of %u -> allocate more; VM_FF_PGM_NO_MEMORY=%RTbool\n",
2114	pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages), VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY) ));
2115	# ifdef IN_RING3
2116	int rc = PGMR3PhysAllocateHandyPages(pVM);
2117	# else
2118	int rc = pgmR0PhysAllocateHandyPages(pVM, VMMGetCpuId(pVM), false /fRing3/);
2119	# endif
2120	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2121	{
2122	if (RT_FAILURE(rc))
2123	return rc;
2124	AssertMsgReturn(rc == VINF_EM_NO_MEMORY, ("%Rrc\n", rc), VERR_IPE_UNEXPECTED_INFO_STATUS);
2125	if (!pVM->pgm.s.cHandyPages)
2126	{
2127	LogRel(("PGM: no more handy pages!\n"));
2128	return VERR_EM_NO_MEMORY;
2129	}
2130	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES));
2131	Assert(VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY));
2132	# ifndef IN_RING3
2133	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3); /* paranoia */
2134	# endif
2135	}
2136	AssertMsgReturn( pVM->pgm.s.cHandyPages > 0
2137	&& pVM->pgm.s.cHandyPages <= RT_ELEMENTS(pVM->pgm.s.aHandyPages),
2138	("%u\n", pVM->pgm.s.cHandyPages),
2139	VERR_PGM_HANDY_PAGE_IPE);
2140	}
2141	else
2142	{
2143	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_SET_FF)
2144	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
2145	# ifndef IN_RING3
2146	if (pVM->pgm.s.cHandyPages <= PGM_HANDY_PAGES_RZ_TO_R3)
2147	{
2148	Log(("PGM: VM_FF_TO_R3 - cHandyPages=%u out of %u\n", pVM->pgm.s.cHandyPages, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
2149	VMCPU_FF_SET(VMMGetCpu(pVM), VMCPU_FF_TO_R3);
2150	}
2151	# endif
2152	}
2153	}
2154
2155	return VINF_SUCCESS;
2156	}
2157
2158
2159	/**
2160	* Replace a zero or shared page with new page that we can write to.
2161	*
2162	* @returns The following VBox status codes.
2163	* @retval VINF_SUCCESS on success, pPage is modified.
2164	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2165	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2166	*
2167	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2168	*
2169	* @param pVM The cross context VM structure.
2170	* @param pPage The physical page tracking structure. This will
2171	* be modified on success.
2172	* @param GCPhys The address of the page.
2173	*
2174	* @remarks Must be called from within the PGM critical section. It may
2175	* nip back to ring-3/0 in some cases.
2176	*
2177	* @remarks This function shouldn't really fail, however if it does
2178	* it probably means we've screwed up the size of handy pages and/or
2179	* the low-water mark. Or, that some device I/O is causing a lot of
2180	* pages to be allocated while while the host is in a low-memory
2181	* condition. This latter should be handled elsewhere and in a more
2182	* controlled manner, it's on the @bugref{3170} todo list...
2183	*/
2184	int pgmPhysAllocPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2185	{
2186	LogFlow(("pgmPhysAllocPage: %R[pgmpage] %RGp\n", pPage, GCPhys));
2187
2188	/*
2189	* Prereqs.
2190	*/
2191	PGM_LOCK_ASSERT_OWNER(pVM);
2192	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2193	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2194
2195	# ifdef PGM_WITH_LARGE_PAGES
2196	/*
2197	* Try allocate a large page if applicable.
2198	*/
2199	if ( PGMIsUsingLargePages(pVM)
2200	&& PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2201	&& !VM_IS_NEM_ENABLED(pVM)) /** @todo NEM: Implement large pages support. */
2202	{
2203	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2204	PPGMPAGE pBasePage;
2205
2206	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pBasePage);
2207	AssertRCReturn(rc, rc); /* paranoia; can't happen. */
2208	if (PGM_PAGE_GET_PDE_TYPE(pBasePage) == PGM_PAGE_PDE_TYPE_DONTCARE)
2209	{
2210	rc = pgmPhysAllocLargePage(pVM, GCPhys);
2211	if (rc == VINF_SUCCESS)
2212	return rc;
2213	}
2214	/* Mark the base as type page table, so we don't check over and over again. */
2215	PGM_PAGE_SET_PDE_TYPE(pVM, pBasePage, PGM_PAGE_PDE_TYPE_PT);
2216
2217	/* fall back to 4KB pages. */
2218	}
2219	# endif
2220
2221	/*
2222	* Flush any shadow page table mappings of the page.
2223	* When VBOX_WITH_NEW_LAZY_PAGE_ALLOC isn't defined, there shouldn't be any.
2224	*/
2225	bool fFlushTLBs = false;
2226	int rc = pgmPoolTrackUpdateGCPhys(pVM, GCPhys, pPage, true /fFlushTLBs/, &fFlushTLBs);
2227	AssertMsgReturn(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3, ("%Rrc\n", rc), RT_FAILURE(rc) ? rc : VERR_IPE_UNEXPECTED_STATUS);
2228
2229	/*
2230	* Ensure that we've got a page handy, take it and use it.
2231	*/
2232	int rc2 = pgmPhysEnsureHandyPage(pVM);
2233	if (RT_FAILURE(rc2))
2234	{
2235	if (fFlushTLBs)
2236	PGM_INVL_ALL_VCPU_TLBS(pVM);
2237	Assert(rc2 == VERR_EM_NO_MEMORY);
2238	return rc2;
2239	}
2240	/* re-assert preconditions since pgmPhysEnsureHandyPage may do a context switch. */
2241	PGM_LOCK_ASSERT_OWNER(pVM);
2242	AssertMsg(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_PAGE_IS_SHARED(pPage), ("%R[pgmpage] %RGp\n", pPage, GCPhys));
2243	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage));
2244
2245	uint32_t iHandyPage = --pVM->pgm.s.cHandyPages;
2246	AssertMsg(iHandyPage < RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d\n", iHandyPage));
2247	Assert(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys != NIL_GMMPAGEDESC_PHYS);
2248	Assert(!(pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys & ~X86_PTE_PAE_PG_MASK));
2249	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idPage != NIL_GMM_PAGEID);
2250	Assert(pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage == NIL_GMM_PAGEID);
2251
2252	/*
2253	* There are one or two action to be taken the next time we allocate handy pages:
2254	* - Tell the GMM (global memory manager) what the page is being used for.
2255	* (Speeds up replacement operations - sharing and defragmenting.)
2256	* - If the current backing is shared, it must be freed.
2257	*/
2258	const RTHCPHYS HCPhys = pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys;
2259	pVM->pgm.s.aHandyPages[iHandyPage].HCPhysGCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
2260
2261	void const *pvSharedPage = NULL;
2262	if (!PGM_PAGE_IS_SHARED(pPage))
2263	{
2264	Log2(("PGM: Replaced zero page %RGp with %#x / %RHp\n", GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2265	STAM_COUNTER_INC(&pVM->pgm.s.Stats.StatRZPageReplaceZero);
2266	pVM->pgm.s.cZeroPages--;
2267	}
2268	else
2269	{
2270	/* Mark this shared page for freeing/dereferencing. */
2271	pVM->pgm.s.aHandyPages[iHandyPage].idSharedPage = PGM_PAGE_GET_PAGEID(pPage);
2272	Assert(PGM_PAGE_GET_PAGEID(pPage) != NIL_GMM_PAGEID);
2273
2274	Log(("PGM: Replaced shared page %#x at %RGp with %#x / %RHp\n", PGM_PAGE_GET_PAGEID(pPage),
2275	GCPhys, pVM->pgm.s.aHandyPages[iHandyPage].idPage, HCPhys));
2276	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageReplaceShared));
2277	pVM->pgm.s.cSharedPages--;
2278
2279	/* Grab the address of the page so we can make a copy later on. (safe) */
2280	rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvSharedPage);
2281	AssertRC(rc);
2282	}
2283
2284	/*
2285	* Do the PGMPAGE modifications.
2286	*/
2287	pVM->pgm.s.cPrivatePages++;
2288	PGM_PAGE_SET_HCPHYS(pVM, pPage, HCPhys);
2289	PGM_PAGE_SET_PAGEID(pVM, pPage, pVM->pgm.s.aHandyPages[iHandyPage].idPage);
2290	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2291	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_PT);
2292	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
2293	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID,
2294	!pvSharedPage
2295	? IEMTLBPHYSFLUSHREASON_ALLOCATED : IEMTLBPHYSFLUSHREASON_ALLOCATED_FROM_SHARED);
2296
2297	/* Copy the shared page contents to the replacement page. */
2298	if (!pvSharedPage)
2299	{ /* likely */ }
2300	else
2301	{
2302	/* Get the virtual address of the new page. */
2303	PGMPAGEMAPLOCK PgMpLck;
2304	void *pvNewPage;
2305	rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvNewPage, &PgMpLck); AssertRC(rc);
2306	if (RT_SUCCESS(rc))
2307	{
2308	memcpy(pvNewPage, pvSharedPage, GUEST_PAGE_SIZE); /** @todo todo write ASMMemCopyPage */
2309	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
2310	}
2311	}
2312
2313	if ( fFlushTLBs
2314	&& rc != VINF_PGM_GCPHYS_ALIASED)
2315	PGM_INVL_ALL_VCPU_TLBS(pVM);
2316
2317	/*
2318	* Notify NEM about the mapping change for this page.
2319	*
2320	* Note! Shadow ROM pages are complicated as they can definitely be
2321	* allocated while not visible, so play safe.
2322	*/
2323	if (VM_IS_NEM_ENABLED(pVM))
2324	{
2325	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2326	if ( enmType != PGMPAGETYPE_ROM_SHADOW
2327	\|\| pgmPhysGetPage(pVM, GCPhys) == pPage)
2328	{
2329	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2330	rc2 = NEMHCNotifyPhysPageAllocated(pVM, GCPhys & ~(RTGCPHYS)X86_PAGE_OFFSET_MASK, HCPhys,
2331	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2332	if (RT_SUCCESS(rc))
2333	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2334	else
2335	rc = rc2;
2336	}
2337	}
2338
2339	return rc;
2340	}
2341
2342	# ifdef PGM_WITH_LARGE_PAGES
2343
2344	/**
2345	* Replace a 2 MB range of zero pages with new pages that we can write to.
2346	*
2347	* @returns The following VBox status codes.
2348	* @retval VINF_SUCCESS on success, pPage is modified.
2349	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2350	* @retval VERR_EM_NO_MEMORY if we're totally out of memory.
2351	*
2352	* @todo Propagate VERR_EM_NO_MEMORY up the call tree.
2353	*
2354	* @param pVM The cross context VM structure.
2355	* @param GCPhys The address of the page.
2356	*
2357	* @remarks Must be called from within the PGM critical section. It may block
2358	* on GMM and host mutexes/locks, leaving HM context.
2359	*/
2360	int pgmPhysAllocLargePage(PVMCC pVM, RTGCPHYS GCPhys)
2361	{
2362	RTGCPHYS GCPhysBase = GCPhys & X86_PDE2M_PAE_PG_MASK;
2363	LogFlow(("pgmPhysAllocLargePage: %RGp base %RGp\n", GCPhys, GCPhysBase));
2364	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2365
2366	/*
2367	* Check Prereqs.
2368	*/
2369	PGM_LOCK_ASSERT_OWNER(pVM);
2370	Assert(PGMIsUsingLargePages(pVM));
2371
2372	/*
2373	* All the pages must be unallocated RAM pages, i.e. mapping the ZERO page.
2374	*/
2375	PPGMPAGE pFirstPage;
2376	int rc = pgmPhysGetPageEx(pVM, GCPhysBase, &pFirstPage);
2377	if ( RT_SUCCESS(rc)
2378	&& PGM_PAGE_GET_TYPE(pFirstPage) == PGMPAGETYPE_RAM
2379	&& PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ZERO)
2380	{
2381	/*
2382	* Further they should have PDE type set to PGM_PAGE_PDE_TYPE_DONTCARE,
2383	* since they are unallocated.
2384	*/
2385	unsigned uPDEType = PGM_PAGE_GET_PDE_TYPE(pFirstPage);
2386	Assert(uPDEType != PGM_PAGE_PDE_TYPE_PDE);
2387	if (uPDEType == PGM_PAGE_PDE_TYPE_DONTCARE)
2388	{
2389	/*
2390	* Now, make sure all the other pages in the 2 MB is in the same state.
2391	*/
2392	GCPhys = GCPhysBase;
2393	unsigned cLeft = _2M / GUEST_PAGE_SIZE;
2394	while (cLeft-- > 0)
2395	{
2396	PPGMPAGE pSubPage = pgmPhysGetPage(pVM, GCPhys);
2397	if ( pSubPage
2398	&& PGM_PAGE_GET_TYPE(pSubPage) == PGMPAGETYPE_RAM /* Anything other than ram implies monitoring. */
2399	&& PGM_PAGE_GET_STATE(pSubPage) == PGM_PAGE_STATE_ZERO) /* Allocated, monitored or shared means we can't use a large page here */
2400	{
2401	Assert(PGM_PAGE_GET_PDE_TYPE(pSubPage) == PGM_PAGE_PDE_TYPE_DONTCARE);
2402	GCPhys += GUEST_PAGE_SIZE;
2403	}
2404	else
2405	{
2406	LogFlow(("pgmPhysAllocLargePage: Found page %RGp with wrong attributes (type=%d; state=%d); cancel check.\n",
2407	GCPhys, pSubPage ? PGM_PAGE_GET_TYPE(pSubPage) : -1, pSubPage ? PGM_PAGE_GET_STATE(pSubPage) : -1));
2408
2409	/* Failed. Mark as requiring a PT so we don't check the whole thing again in the future. */
2410	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRefused);
2411	PGM_PAGE_SET_PDE_TYPE(pVM, pFirstPage, PGM_PAGE_PDE_TYPE_PT);
2412	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2413	}
2414	}
2415
2416	/*
2417	* Do the allocation.
2418	*/
2419	# ifdef IN_RING3
2420	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_LARGE_PAGE, GCPhysBase, NULL);
2421	# elif defined(IN_RING0)
2422	rc = pgmR0PhysAllocateLargePage(pVM, VMMGetCpuId(pVM), GCPhysBase);
2423	# else
2424	# error "Port me"
2425	# endif
2426	if (RT_SUCCESS(rc))
2427	{
2428	Assert(PGM_PAGE_GET_STATE(pFirstPage) == PGM_PAGE_STATE_ALLOCATED);
2429	pVM->pgm.s.cLargePages++;
2430	return VINF_SUCCESS;
2431	}
2432
2433	/* If we fail once, it most likely means the host's memory is too
2434	fragmented; don't bother trying again. */
2435	LogFlow(("pgmPhysAllocLargePage failed with %Rrc\n", rc));
2436	return rc;
2437	}
2438	}
2439	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2440	}
2441
2442
2443	/**
2444	* Recheck the entire 2 MB range to see if we can use it again as a large page.
2445	*
2446	* @returns The following VBox status codes.
2447	* @retval VINF_SUCCESS on success, the large page can be used again
2448	* @retval VERR_PGM_INVALID_LARGE_PAGE_RANGE if it can't be reused
2449	*
2450	* @param pVM The cross context VM structure.
2451	* @param GCPhys The address of the page.
2452	* @param pLargePage Page structure of the base page
2453	*/
2454	int pgmPhysRecheckLargePage(PVMCC pVM, RTGCPHYS GCPhys, PPGMPAGE pLargePage)
2455	{
2456	STAM_REL_COUNTER_INC(&pVM->pgm.s.StatLargePageRecheck);
2457
2458	Assert(!VM_IS_NEM_ENABLED(pVM)); /** @todo NEM: Large page support. */
2459
2460	AssertCompile(X86_PDE2M_PAE_PG_MASK == EPT_PDE2M_PG_MASK); /* Paranoia: Caller uses this for guest EPT tables as well. */
2461	GCPhys &= X86_PDE2M_PAE_PG_MASK;
2462
2463	/* Check the base page. */
2464	Assert(PGM_PAGE_GET_PDE_TYPE(pLargePage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED);
2465	if ( PGM_PAGE_GET_STATE(pLargePage) != PGM_PAGE_STATE_ALLOCATED
2466	\|\| PGM_PAGE_GET_TYPE(pLargePage) != PGMPAGETYPE_RAM
2467	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2468	{
2469	LogFlow(("pgmPhysRecheckLargePage: checks failed for base page %x %x %x\n", PGM_PAGE_GET_STATE(pLargePage), PGM_PAGE_GET_TYPE(pLargePage), PGM_PAGE_GET_HNDL_PHYS_STATE(pLargePage)));
2470	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2471	}
2472
2473	STAM_PROFILE_START(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2474	/* Check all remaining pages in the 2 MB range. */
2475	unsigned i;
2476	GCPhys += GUEST_PAGE_SIZE;
2477	for (i = 1; i < _2M / GUEST_PAGE_SIZE; i++)
2478	{
2479	PPGMPAGE pPage;
2480	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
2481	AssertRCBreak(rc);
2482
2483	if ( PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
2484	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
2485	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
2486	\|\| PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) != PGM_PAGE_HNDL_PHYS_STATE_NONE)
2487	{
2488	LogFlow(("pgmPhysRecheckLargePage: checks failed for page %d; %x %x %x\n", i, PGM_PAGE_GET_STATE(pPage), PGM_PAGE_GET_TYPE(pPage), PGM_PAGE_GET_HNDL_PHYS_STATE(pPage)));
2489	break;
2490	}
2491
2492	GCPhys += GUEST_PAGE_SIZE;
2493	}
2494	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,IsValidLargePage), a);
2495
2496	if (i == _2M / GUEST_PAGE_SIZE)
2497	{
2498	PGM_PAGE_SET_PDE_TYPE(pVM, pLargePage, PGM_PAGE_PDE_TYPE_PDE);
2499	pVM->pgm.s.cLargePagesDisabled--;
2500	Log(("pgmPhysRecheckLargePage: page %RGp can be reused!\n", GCPhys - _2M));
2501	return VINF_SUCCESS;
2502	}
2503
2504	return VERR_PGM_INVALID_LARGE_PAGE_RANGE;
2505	}
2506
2507	# endif /* PGM_WITH_LARGE_PAGES */
2508	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2509
2510
2511
2512	/**
2513	* Deal with a write monitored page.
2514	*
2515	* @param pVM The cross context VM structure.
2516	* @param pPage The physical page tracking structure.
2517	* @param GCPhys The guest physical address of the page.
2518	* PGMPhysReleasePageMappingLock() passes NIL_RTGCPHYS in a
2519	* very unlikely situation where it is okay that we let NEM
2520	* fix the page access in a lazy fasion.
2521	*
2522	* @remarks Called from within the PGM critical section.
2523	*/
2524	void pgmPhysPageMakeWriteMonitoredWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2525	{
2526	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED);
2527	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
2528	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ALLOCATED);
2529	if (PGM_PAGE_IS_CODE_PAGE(pPage))
2530	{
2531	PGM_PAGE_CLEAR_CODE_PAGE(pVM, pPage);
2532	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_MADE_WRITABLE);
2533	}
2534
2535	Assert(pVM->pgm.s.cMonitoredPages > 0);
2536	pVM->pgm.s.cMonitoredPages--;
2537	pVM->pgm.s.cWrittenToPages++;
2538
2539	#ifdef VBOX_WITH_NATIVE_NEM
2540	/*
2541	* Notify NEM about the protection change so we won't spin forever.
2542	*
2543	* Note! NEM need to be handle to lazily correct page protection as we cannot
2544	* really get it 100% right here it seems. The page pool does this too.
2545	*/
2546	if (VM_IS_NEM_ENABLED(pVM) && GCPhys != NIL_RTGCPHYS)
2547	{
2548	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
2549	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
2550	PPGMRAMRANGE pRam = pgmPhysGetRange(pVM, GCPhys);
2551	NEMHCNotifyPhysPageProtChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pPage),
2552	pRam ? PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhys) : NULL,
2553	pgmPhysPageCalcNemProtection(pPage, enmType), enmType, &u2State);
2554	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
2555	}
2556	#else
2557	RT_NOREF(GCPhys);
2558	#endif
2559	}
2560
2561
2562	/**
2563	* Deal with pages that are not writable, i.e. not in the ALLOCATED state.
2564	*
2565	* @returns VBox strict status code.
2566	* @retval VINF_SUCCESS on success.
2567	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2568	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2569	*
2570	* @param pVM The cross context VM structure.
2571	* @param pPage The physical page tracking structure.
2572	* @param GCPhys The address of the page.
2573	*
2574	* @remarks Called from within the PGM critical section.
2575	*/
2576	int pgmPhysPageMakeWritable(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2577	{
2578	PGM_LOCK_ASSERT_OWNER(pVM);
2579	switch (PGM_PAGE_GET_STATE(pPage))
2580	{
2581	case PGM_PAGE_STATE_WRITE_MONITORED:
2582	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
2583	RT_FALL_THRU();
2584	default: /* to shut up GCC */
2585	case PGM_PAGE_STATE_ALLOCATED:
2586	return VINF_SUCCESS;
2587
2588	/*
2589	* Zero pages can be dummy pages for MMIO or reserved memory,
2590	* so we need to check the flags before joining cause with
2591	* shared page replacement.
2592	*/
2593	case PGM_PAGE_STATE_ZERO:
2594	if (PGM_PAGE_IS_MMIO(pPage))
2595	return VERR_PGM_PHYS_PAGE_RESERVED;
2596	RT_FALL_THRU();
2597	case PGM_PAGE_STATE_SHARED:
2598	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2599	return pgmPhysAllocPage(pVM, pPage, GCPhys);
2600	#else
2601	AssertFailed(); /** @todo not sure if we make use of ZERO pages or not in NEM-mode, but I can't see how pgmPhysAllocPage would work. */
2602	return VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE;
2603	#endif
2604
2605	/* Not allowed to write to ballooned pages. */
2606	case PGM_PAGE_STATE_BALLOONED:
2607	return VERR_PGM_PHYS_PAGE_BALLOONED;
2608	}
2609	}
2610
2611	#if 0 /* unused */
2612	/**
2613	* Internal usage: Map the page specified by its GMM ID.
2614	*
2615	* This is similar to pgmPhysPageMap
2616	*
2617	* @returns VBox status code.
2618	*
2619	* @param pVM The cross context VM structure.
2620	* @param idPage The Page ID.
2621	* @param HCPhys The physical address (for SUPR0HCPhysToVirt).
2622	* @param ppv Where to store the mapping address.
2623	*
2624	* @remarks Called from within the PGM critical section. The mapping is only
2625	* valid while you are inside this section.
2626	*/
2627	int pgmPhysPageMapByPageID(PVMCC pVM, uint32_t idPage, RTHCPHYS HCPhys, void **ppv)
2628	{
2629	/*
2630	* Validation.
2631	*/
2632	PGM_LOCK_ASSERT_OWNER(pVM);
2633	AssertReturn(HCPhys && !(HCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2634	const uint32_t idChunk = idPage >> GMM_CHUNKID_SHIFT;
2635	AssertReturn(idChunk != NIL_GMM_CHUNKID, VERR_INVALID_PARAMETER);
2636
2637	#ifdef IN_RING0
2638	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2639	return SUPR0HCPhysToVirt(HCPhys & ~(RTHCPHYS)GUEST_PAGE_OFFSET_MASK, ppv);
2640	# else
2641	return GMMR0PageIdToVirt(pVM, idPage, ppv);
2642	# endif
2643
2644	#else
2645	/*
2646	* Find/make Chunk TLB entry for the mapping chunk.
2647	*/
2648	PPGMCHUNKR3MAP pMap;
2649	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2650	if (pTlbe->idChunk == idChunk)
2651	{
2652	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2653	pMap = pTlbe->pChunk;
2654	}
2655	else
2656	{
2657	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2658
2659	/*
2660	* Find the chunk, map it if necessary.
2661	*/
2662	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2663	if (pMap)
2664	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2665	else
2666	{
2667	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2668	if (RT_FAILURE(rc))
2669	return rc;
2670	}
2671
2672	/*
2673	* Enter it into the Chunk TLB.
2674	*/
2675	pTlbe->idChunk = idChunk;
2676	pTlbe->pChunk = pMap;
2677	}
2678
2679	ppv = (uint8_t )pMap->pv + ((idPage & GMM_PAGEID_IDX_MASK) << GUEST_PAGE_SHIFT);
2680	return VINF_SUCCESS;
2681	#endif
2682	}
2683	#endif /* unused */
2684
2685	/**
2686	* Maps a page into the current virtual address space so it can be accessed.
2687	*
2688	* @returns VBox status code.
2689	* @retval VINF_SUCCESS on success.
2690	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2691	*
2692	* @param pVM The cross context VM structure.
2693	* @param pPage The physical page tracking structure.
2694	* @param GCPhys The address of the page.
2695	* @param ppMap Where to store the address of the mapping tracking structure.
2696	* @param ppv Where to store the mapping address of the page. The page
2697	* offset is masked off!
2698	*
2699	* @remarks Called from within the PGM critical section.
2700	*/
2701	static int pgmPhysPageMapCommon(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, PPPGMPAGEMAP ppMap, void **ppv)
2702	{
2703	PGM_LOCK_ASSERT_OWNER(pVM);
2704	NOREF(GCPhys);
2705
2706	/*
2707	* Special cases: MMIO2 and specially aliased MMIO pages.
2708	*/
2709	if ( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2
2710	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO)
2711	{
2712	*ppMap = NULL;
2713
2714	/* Decode the page id to a page in a MMIO2 ram range. */
2715	uint8_t const idMmio2 = PGM_MMIO2_PAGEID_GET_MMIO2_ID(PGM_PAGE_GET_PAGEID(pPage));
2716	uint32_t const iPage = PGM_MMIO2_PAGEID_GET_IDX(PGM_PAGE_GET_PAGEID(pPage));
2717	AssertLogRelMsgReturn((uint8_t)(idMmio2 - 1U) < RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges),
2718	("idMmio2=%u size=%u type=%u GCPHys=%#RGp Id=%u State=%u", idMmio2,
2719	RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges), PGM_PAGE_GET_TYPE(pPage), GCPhys,
2720	pPage->s.idPage, pPage->s.uStateY),
2721	VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2722	PPGMREGMMIO2RANGE const pMmio2Range = &pVM->pgm.s.aMmio2Ranges[idMmio2 - 1];
2723	AssertLogRelReturn(pMmio2Range, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2724	AssertLogRelReturn(pMmio2Range->idMmio2 == idMmio2, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2725	#ifndef IN_RING0
2726	uint32_t const idRamRange = pMmio2Range->idRamRange;
2727	AssertLogRelReturn(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2728	PPGMRAMRANGE const pRamRange = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
2729	AssertLogRelReturn(pRamRange, VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2730	AssertLogRelReturn(iPage < (pRamRange->cb >> GUEST_PAGE_SHIFT), VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2731	*ppv = pMmio2Range->pbR3 + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2732	return VINF_SUCCESS;
2733
2734	#else /* IN_RING0 */
2735	AssertLogRelReturn(iPage < pVM->pgmr0.s.acMmio2RangePages[idMmio2 - 1], VERR_PGM_PHYS_PAGE_MAP_MMIO2_IPE);
2736	# ifdef VBOX_WITH_LINEAR_HOST_PHYS_MEM
2737	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2738	# else
2739	AssertPtr(pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1]);
2740	*ppv = pVM->pgmr0.s.apbMmio2Backing[idMmio2 - 1] + ((uintptr_t)iPage << GUEST_PAGE_SHIFT);
2741	return VINF_SUCCESS;
2742	# endif
2743	#endif
2744	}
2745
2746	#ifdef VBOX_WITH_PGM_NEM_MODE
2747	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2748	if (pVM->pgm.s.fNemMode)
2749	# endif
2750	{
2751	# ifdef IN_RING3
2752	/*
2753	* Find the corresponding RAM range and use that to locate the mapping address.
2754	*/
2755	/** @todo Use the page ID for some kind of indexing as we do with MMIO2 above. */
2756	PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
2757	AssertLogRelMsgReturn(pRam, ("%RTGp\n", GCPhys), VERR_INTERNAL_ERROR_3);
2758	size_t const idxPage = (GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT;
2759	Assert(pPage == &pRam->aPages[idxPage]);
2760	*ppMap = NULL;
2761	ppv = (uint8_t )pRam->pbR3 + (idxPage << GUEST_PAGE_SHIFT);
2762	return VINF_SUCCESS;
2763	# else
2764	AssertFailedReturn(VERR_INTERNAL_ERROR_2);
2765	# endif
2766	}
2767	#endif /* VBOX_WITH_PGM_NEM_MODE */
2768	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
2769
2770	const uint32_t idChunk = PGM_PAGE_GET_CHUNKID(pPage);
2771	if (idChunk == NIL_GMM_CHUNKID)
2772	{
2773	AssertMsgReturn(PGM_PAGE_GET_PAGEID(pPage) == NIL_GMM_PAGEID, ("pPage=%R[pgmpage]\n", pPage),
2774	VERR_PGM_PHYS_PAGE_MAP_IPE_1);
2775	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
2776	{
2777	AssertMsgReturn(PGM_PAGE_IS_ZERO(pPage), ("pPage=%R[pgmpage]\n", pPage),
2778	VERR_PGM_PHYS_PAGE_MAP_IPE_3);
2779	AssertMsgReturn(PGM_PAGE_GET_HCPHYS(pPage)== pVM->pgm.s.HCPhysZeroPg, ("pPage=%R[pgmpage]\n", pPage),
2780	VERR_PGM_PHYS_PAGE_MAP_IPE_4);
2781	*ppv = pVM->pgm.s.abZeroPg;
2782	}
2783	else
2784	*ppv = pVM->pgm.s.abZeroPg;
2785	*ppMap = NULL;
2786	return VINF_SUCCESS;
2787	}
2788
2789	# if defined(IN_RING0) && defined(VBOX_WITH_LINEAR_HOST_PHYS_MEM)
2790	/*
2791	* Just use the physical address.
2792	*/
2793	*ppMap = NULL;
2794	return SUPR0HCPhysToVirt(PGM_PAGE_GET_HCPHYS(pPage), ppv);
2795
2796	# elif defined(IN_RING0)
2797	/*
2798	* Go by page ID thru GMMR0.
2799	*/
2800	*ppMap = NULL;
2801	return GMMR0PageIdToVirt(pVM, PGM_PAGE_GET_PAGEID(pPage), ppv);
2802
2803	# else
2804	/*
2805	* Find/make Chunk TLB entry for the mapping chunk.
2806	*/
2807	PPGMCHUNKR3MAP pMap;
2808	PPGMCHUNKR3MAPTLBE pTlbe = &pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(idChunk)];
2809	if (pTlbe->idChunk == idChunk)
2810	{
2811	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbHits));
2812	pMap = pTlbe->pChunk;
2813	AssertPtr(pMap->pv);
2814	}
2815	else
2816	{
2817	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,ChunkR3MapTlbMisses));
2818
2819	/*
2820	* Find the chunk, map it if necessary.
2821	*/
2822	pMap = (PPGMCHUNKR3MAP)RTAvlU32Get(&pVM->pgm.s.ChunkR3Map.pTree, idChunk);
2823	if (pMap)
2824	{
2825	AssertPtr(pMap->pv);
2826	pMap->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
2827	}
2828	else
2829	{
2830	int rc = pgmR3PhysChunkMap(pVM, idChunk, &pMap);
2831	if (RT_FAILURE(rc))
2832	return rc;
2833	AssertPtr(pMap->pv);
2834	}
2835
2836	/*
2837	* Enter it into the Chunk TLB.
2838	*/
2839	pTlbe->idChunk = idChunk;
2840	pTlbe->pChunk = pMap;
2841	}
2842
2843	ppv = (uint8_t )pMap->pv + (PGM_PAGE_GET_PAGE_IN_CHUNK(pPage) << GUEST_PAGE_SHIFT);
2844	*ppMap = pMap;
2845	return VINF_SUCCESS;
2846	# endif /* !IN_RING0 */
2847	#endif /* !VBOX_WITH_ONLY_PGM_NEM_MODE */
2848	}
2849
2850
2851	/**
2852	* Combination of pgmPhysPageMakeWritable and pgmPhysPageMapWritable.
2853	*
2854	* This is typically used is paths where we cannot use the TLB methods (like ROM
2855	* pages) or where there is no point in using them since we won't get many hits.
2856	*
2857	* @returns VBox strict status code.
2858	* @retval VINF_SUCCESS on success.
2859	* @retval VINF_PGM_SYNC_CR3 on success and a page pool flush is pending.
2860	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2861	*
2862	* @param pVM The cross context VM structure.
2863	* @param pPage The physical page tracking structure.
2864	* @param GCPhys The address of the page.
2865	* @param ppv Where to store the mapping address of the page. The page
2866	* offset is masked off!
2867	*
2868	* @remarks Called from within the PGM critical section. The mapping is only
2869	* valid while you are inside section.
2870	*/
2871	int pgmPhysPageMakeWritableAndMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2872	{
2873	int rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
2874	if (RT_SUCCESS(rc))
2875	{
2876	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* returned */, ("%Rrc\n", rc));
2877	PPGMPAGEMAP pMapIgnore;
2878	int rc2 = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2879	if (RT_FAILURE(rc2)) /* preserve rc */
2880	rc = rc2;
2881	}
2882	return rc;
2883	}
2884
2885
2886	/**
2887	* Maps a page into the current virtual address space so it can be accessed for
2888	* both writing and reading.
2889	*
2890	* This is typically used is paths where we cannot use the TLB methods (like ROM
2891	* pages) or where there is no point in using them since we won't get many hits.
2892	*
2893	* @returns VBox status code.
2894	* @retval VINF_SUCCESS on success.
2895	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2896	*
2897	* @param pVM The cross context VM structure.
2898	* @param pPage The physical page tracking structure. Must be in the
2899	* allocated state.
2900	* @param GCPhys The address of the page.
2901	* @param ppv Where to store the mapping address of the page. The page
2902	* offset is masked off!
2903	*
2904	* @remarks Called from within the PGM critical section. The mapping is only
2905	* valid while you are inside section.
2906	*/
2907	int pgmPhysPageMap(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
2908	{
2909	Assert(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED);
2910	PPGMPAGEMAP pMapIgnore;
2911	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, ppv);
2912	}
2913
2914
2915	/**
2916	* Maps a page into the current virtual address space so it can be accessed for
2917	* reading.
2918	*
2919	* This is typically used is paths where we cannot use the TLB methods (like ROM
2920	* pages) or where there is no point in using them since we won't get many hits.
2921	*
2922	* @returns VBox status code.
2923	* @retval VINF_SUCCESS on success.
2924	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
2925	*
2926	* @param pVM The cross context VM structure.
2927	* @param pPage The physical page tracking structure.
2928	* @param GCPhys The address of the page.
2929	* @param ppv Where to store the mapping address of the page. The page
2930	* offset is masked off!
2931	*
2932	* @remarks Called from within the PGM critical section. The mapping is only
2933	* valid while you are inside this section.
2934	*/
2935	int pgmPhysPageMapReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const **ppv)
2936	{
2937	PPGMPAGEMAP pMapIgnore;
2938	return pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMapIgnore, (void **)ppv);
2939	}
2940
2941
2942	/**
2943	* Load a guest page into the ring-3 physical TLB.
2944	*
2945	* @returns VBox status code.
2946	* @retval VINF_SUCCESS on success
2947	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2948	* @param pVM The cross context VM structure.
2949	* @param GCPhys The guest physical address in question.
2950	*/
2951	int pgmPhysPageLoadIntoTlb(PVMCC pVM, RTGCPHYS GCPhys)
2952	{
2953	PGM_LOCK_ASSERT_OWNER(pVM);
2954
2955	/*
2956	* Find the ram range and page and hand it over to the with-page function.
2957	* 99.8% of requests are expected to be in the first range.
2958	*/
2959	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
2960	if (!pPage)
2961	{
2962	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2963	return VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS;
2964	}
2965
2966	return pgmPhysPageLoadIntoTlbWithPage(pVM, pPage, GCPhys);
2967	}
2968
2969
2970	/**
2971	* Load a guest page into the ring-3 physical TLB.
2972	*
2973	* @returns VBox status code.
2974	* @retval VINF_SUCCESS on success
2975	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
2976	*
2977	* @param pVM The cross context VM structure.
2978	* @param pPage Pointer to the PGMPAGE structure corresponding to
2979	* GCPhys.
2980	* @param GCPhys The guest physical address in question.
2981	*/
2982	int pgmPhysPageLoadIntoTlbWithPage(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys)
2983	{
2984	PGM_LOCK_ASSERT_OWNER(pVM);
2985	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PageMapTlbMisses));
2986
2987	/*
2988	* Map the page.
2989	* Make a special case for the zero page as it is kind of special.
2990	*/
2991	PPGMPAGEMAPTLBE pTlbe = &pVM->pgm.s.CTX_SUFF(PhysTlb).aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
2992	if ( !PGM_PAGE_IS_ZERO(pPage)
2993	&& !PGM_PAGE_IS_BALLOONED(pPage))
2994	{
2995	void *pv;
2996	PPGMPAGEMAP pMap;
2997	int rc = pgmPhysPageMapCommon(pVM, pPage, GCPhys, &pMap, &pv);
2998	if (RT_FAILURE(rc))
2999	return rc;
3000	#ifndef IN_RING0
3001	pTlbe->pMap = pMap;
3002	#endif
3003	pTlbe->pv = pv;
3004	Assert(!((uintptr_t)pTlbe->pv & GUEST_PAGE_OFFSET_MASK));
3005	}
3006	else
3007	{
3008	#ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3009	AssertMsg(PGM_PAGE_GET_HCPHYS(pPage) == pVM->pgm.s.HCPhysZeroPg, ("%RGp/%R[pgmpage]\n", GCPhys, pPage));
3010	#ifndef IN_RING0
3011	pTlbe->pMap = NULL;
3012	#endif
3013	pTlbe->pv = pVM->pgm.s.abZeroPg;
3014	#else
3015	/*
3016	* Should not ever be used, as we don't implement zero pages for NEM mode currently and
3017	* MMIO accesses are not handled through the TLB.
3018	*/
3019	pTlbe->pv = (void )&g_abRTZero64K[0]; / Maximum granule size on ARM. */
3020	#endif
3021	}
3022	#ifdef PGM_WITH_PHYS_TLB
3023	if ( PGM_PAGE_GET_TYPE(pPage) < PGMPAGETYPE_ROM_SHADOW
3024	\|\| PGM_PAGE_GET_TYPE(pPage) > PGMPAGETYPE_ROM)
3025	pTlbe->GCPhys = GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3026	else
3027	pTlbe->GCPhys = NIL_RTGCPHYS; /* ROM: Problematic because of the two pages. :-/ */
3028	#else
3029	pTlbe->GCPhys = NIL_RTGCPHYS;
3030	#endif
3031	pTlbe->pPage = pPage;
3032	return VINF_SUCCESS;
3033	}
3034
3035
3036	#ifdef IN_RING3 /** @todo Need ensure a ring-0 version gets invalidated safely */
3037	/**
3038	* Load a guest page into the lockless ring-3 physical TLB for the calling EMT.
3039	*
3040	* @returns VBox status code.
3041	* @retval VINF_SUCCESS on success
3042	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3043	*
3044	* @param pVCpu The cross context virtual CPU structure.
3045	* @param pPage Pointer to the PGMPAGE structure corresponding to
3046	* GCPhys.
3047	* @param GCPhys The guest physical address in question.
3048	*/
3049	DECLHIDDEN(int) pgmPhysPageLoadIntoLocklessTlbWithPage(PVMCPUCC pVCpu, PPGMPAGE pPage, RTGCPHYS GCPhys)
3050	{
3051	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.CTX_MID_Z(Stat,PageMapTlbMisses));
3052	PPGMPAGEMAPTLBE const pLocklessTlbe = &pVCpu->pgm.s.PhysTlb.aEntries[PGM_PAGEMAPTLB_IDX(GCPhys)];
3053	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3054
3055	PGM_LOCK_VOID(pVM);
3056
3057	PPGMPAGEMAPTLBE pSharedTlbe;
3058	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pSharedTlbe);
3059	if (RT_SUCCESS(rc))
3060	pLocklessTlbe = pSharedTlbe;
3061
3062	PGM_UNLOCK(pVM);
3063	return rc;
3064	}
3065	#endif /* IN_RING3 */
3066
3067
3068	/**
3069	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3070	* own the PGM lock and therefore not need to lock the mapped page.
3071	*
3072	* @returns VBox status code.
3073	* @retval VINF_SUCCESS on success.
3074	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3075	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3076	*
3077	* @param pVM The cross context VM structure.
3078	* @param GCPhys The guest physical address of the page that should be mapped.
3079	* @param pPage Pointer to the PGMPAGE structure for the page.
3080	* @param ppv Where to store the address corresponding to GCPhys.
3081	*
3082	* @internal
3083	* @deprecated Use pgmPhysGCPhys2CCPtrInternalEx.
3084	*/
3085	int pgmPhysGCPhys2CCPtrInternalDepr(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv)
3086	{
3087	int rc;
3088	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3089	PGM_LOCK_ASSERT_OWNER(pVM);
3090	pVM->pgm.s.cDeprecatedPageLocks++;
3091
3092	/*
3093	* Make sure the page is writable.
3094	*/
3095	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3096	{
3097	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3098	if (RT_FAILURE(rc))
3099	return rc;
3100	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3101	}
3102	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3103
3104	/*
3105	* Get the mapping address.
3106	*/
3107	PPGMPAGEMAPTLBE pTlbe;
3108	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3109	if (RT_FAILURE(rc))
3110	return rc;
3111	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3112	return VINF_SUCCESS;
3113	}
3114
3115
3116	/**
3117	* Locks a page mapping for writing.
3118	*
3119	* @param pVM The cross context VM structure.
3120	* @param pPage The page.
3121	* @param pTlbe The mapping TLB entry for the page.
3122	* @param pLock The lock structure (output).
3123	*/
3124	DECLINLINE(void) pgmPhysPageMapLockForWriting(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3125	{
3126	# ifndef IN_RING0
3127	PPGMPAGEMAP pMap = pTlbe->pMap;
3128	if (pMap)
3129	pMap->cRefs++;
3130	# else
3131	RT_NOREF(pTlbe);
3132	# endif
3133
3134	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3135	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3136	{
3137	if (cLocks == 0)
3138	pVM->pgm.s.cWriteLockedPages++;
3139	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3140	}
3141	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3142	{
3143	PGM_PAGE_INC_WRITE_LOCKS(pPage);
3144	AssertMsgFailed(("%R[pgmpage] is entering permanent write locked state!\n", pPage));
3145	# ifndef IN_RING0
3146	if (pMap)
3147	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3148	# endif
3149	}
3150
3151	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
3152	# ifndef IN_RING0
3153	pLock->pvMap = pMap;
3154	# else
3155	pLock->pvMap = NULL;
3156	# endif
3157	}
3158
3159	/**
3160	* Locks a page mapping for reading.
3161	*
3162	* @param pVM The cross context VM structure.
3163	* @param pPage The page.
3164	* @param pTlbe The mapping TLB entry for the page.
3165	* @param pLock The lock structure (output).
3166	*/
3167	DECLINLINE(void) pgmPhysPageMapLockForReading(PVM pVM, PPGMPAGE pPage, PPGMPAGEMAPTLBE pTlbe, PPGMPAGEMAPLOCK pLock)
3168	{
3169	# ifndef IN_RING0
3170	PPGMPAGEMAP pMap = pTlbe->pMap;
3171	if (pMap)
3172	pMap->cRefs++;
3173	# else
3174	RT_NOREF(pTlbe);
3175	# endif
3176
3177	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3178	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
3179	{
3180	if (cLocks == 0)
3181	pVM->pgm.s.cReadLockedPages++;
3182	PGM_PAGE_INC_READ_LOCKS(pPage);
3183	}
3184	else if (cLocks != PGM_PAGE_MAX_LOCKS)
3185	{
3186	PGM_PAGE_INC_READ_LOCKS(pPage);
3187	AssertMsgFailed(("%R[pgmpage] is entering permanent read locked state!\n", pPage));
3188	# ifndef IN_RING0
3189	if (pMap)
3190	pMap->cRefs++; /* Extra ref to prevent it from going away. */
3191	# endif
3192	}
3193
3194	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
3195	# ifndef IN_RING0
3196	pLock->pvMap = pMap;
3197	# else
3198	pLock->pvMap = NULL;
3199	# endif
3200	}
3201
3202
3203	/**
3204	* Internal version of PGMPhysGCPhys2CCPtr that expects the caller to
3205	* own the PGM lock and have access to the page structure.
3206	*
3207	* @returns VBox status code.
3208	* @retval VINF_SUCCESS on success.
3209	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3210	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3211	*
3212	* @param pVM The cross context VM structure.
3213	* @param GCPhys The guest physical address of the page that should be mapped.
3214	* @param pPage Pointer to the PGMPAGE structure for the page.
3215	* @param ppv Where to store the address corresponding to GCPhys.
3216	* @param pLock Where to store the lock information that
3217	* pgmPhysReleaseInternalPageMappingLock needs.
3218	*
3219	* @internal
3220	*/
3221	int pgmPhysGCPhys2CCPtrInternal(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3222	{
3223	int rc;
3224	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3225	PGM_LOCK_ASSERT_OWNER(pVM);
3226
3227	/*
3228	* Make sure the page is writable.
3229	*/
3230	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3231	{
3232	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3233	if (RT_FAILURE(rc))
3234	return rc;
3235	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3236	}
3237	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3238
3239	/*
3240	* Do the job.
3241	*/
3242	PPGMPAGEMAPTLBE pTlbe;
3243	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3244	if (RT_FAILURE(rc))
3245	return rc;
3246	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3247	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3248	return VINF_SUCCESS;
3249	}
3250
3251
3252	/**
3253	* Internal version of PGMPhysGCPhys2CCPtrReadOnly that expects the caller to
3254	* own the PGM lock and have access to the page structure.
3255	*
3256	* @returns VBox status code.
3257	* @retval VINF_SUCCESS on success.
3258	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3259	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3260	*
3261	* @param pVM The cross context VM structure.
3262	* @param GCPhys The guest physical address of the page that should be mapped.
3263	* @param pPage Pointer to the PGMPAGE structure for the page.
3264	* @param ppv Where to store the address corresponding to GCPhys.
3265	* @param pLock Where to store the lock information that
3266	* pgmPhysReleaseInternalPageMappingLock needs.
3267	*
3268	* @internal
3269	*/
3270	int pgmPhysGCPhys2CCPtrInternalReadOnly(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, const void **ppv, PPGMPAGEMAPLOCK pLock)
3271	{
3272	AssertReturn(pPage, VERR_PGM_PHYS_NULL_PAGE_PARAM);
3273	PGM_LOCK_ASSERT_OWNER(pVM);
3274	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3275
3276	/*
3277	* Do the job.
3278	*/
3279	PPGMPAGEMAPTLBE pTlbe;
3280	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3281	if (RT_FAILURE(rc))
3282	return rc;
3283	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3284	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3285	return VINF_SUCCESS;
3286	}
3287
3288
3289	/**
3290	* Requests the mapping of a guest page into the current context.
3291	*
3292	* This API should only be used for very short term, as it will consume scarse
3293	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3294	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3295	*
3296	* This API will assume your intention is to write to the page, and will
3297	* therefore replace shared and zero pages. If you do not intend to modify
3298	* the page, use the PGMPhysGCPhys2CCPtrReadOnly() API.
3299	*
3300	* @returns VBox status code.
3301	* @retval VINF_SUCCESS on success.
3302	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3303	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3304	*
3305	* @param pVM The cross context VM structure.
3306	* @param GCPhys The guest physical address of the page that should be
3307	* mapped.
3308	* @param ppv Where to store the address corresponding to GCPhys.
3309	* @param pLock Where to store the lock information that
3310	* PGMPhysReleasePageMappingLock needs.
3311	*
3312	* @remarks The caller is responsible for dealing with access handlers.
3313	* @todo Add an informational return code for pages with access handlers?
3314	*
3315	* @remark Avoid calling this API from within critical sections (other than
3316	* the PGM one) because of the deadlock risk. External threads may
3317	* need to delegate jobs to the EMTs.
3318	* @remarks Only one page is mapped! Make no assumption about what's after or
3319	* before the returned page!
3320	* @thread Any thread.
3321	*/
3322	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtr(PVMCC pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
3323	{
3324	int rc = PGM_LOCK(pVM);
3325	AssertRCReturn(rc, rc);
3326
3327	/*
3328	* Query the Physical TLB entry for the page (may fail).
3329	*/
3330	PPGMPAGEMAPTLBE pTlbe;
3331	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3332	if (RT_SUCCESS(rc))
3333	{
3334	/*
3335	* If the page is shared, the zero page, or being write monitored
3336	* it must be converted to a page that's writable if possible.
3337	*/
3338	PPGMPAGE pPage = pTlbe->pPage;
3339	if (RT_UNLIKELY(PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED))
3340	{
3341	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
3342	if (RT_SUCCESS(rc))
3343	{
3344	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3345	rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
3346	}
3347	}
3348	if (RT_SUCCESS(rc))
3349	{
3350	/*
3351	* Now, just perform the locking and calculate the return address.
3352	*/
3353	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
3354	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3355	}
3356	}
3357
3358	PGM_UNLOCK(pVM);
3359	return rc;
3360	}
3361
3362
3363	/**
3364	* Requests the mapping of a guest page into the current context.
3365	*
3366	* This API should only be used for very short term, as it will consume scarse
3367	* resources (R0 and GC) in the mapping cache. When you're done with the page,
3368	* call PGMPhysReleasePageMappingLock() ASAP to release it.
3369	*
3370	* @returns VBox status code.
3371	* @retval VINF_SUCCESS on success.
3372	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3373	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3374	*
3375	* @param pVM The cross context VM structure.
3376	* @param GCPhys The guest physical address of the page that should be
3377	* mapped.
3378	* @param ppv Where to store the address corresponding to GCPhys.
3379	* @param pLock Where to store the lock information that
3380	* PGMPhysReleasePageMappingLock needs.
3381	*
3382	* @remarks The caller is responsible for dealing with access handlers.
3383	* @todo Add an informational return code for pages with access handlers?
3384	*
3385	* @remarks Avoid calling this API from within critical sections (other than
3386	* the PGM one) because of the deadlock risk.
3387	* @remarks Only one page is mapped! Make no assumption about what's after or
3388	* before the returned page!
3389	* @thread Any thread.
3390	*/
3391	VMM_INT_DECL(int) PGMPhysGCPhys2CCPtrReadOnly(PVMCC pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
3392	{
3393	int rc = PGM_LOCK(pVM);
3394	AssertRCReturn(rc, rc);
3395
3396	/*
3397	* Query the Physical TLB entry for the page (may fail).
3398	*/
3399	PPGMPAGEMAPTLBE pTlbe;
3400	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
3401	if (RT_SUCCESS(rc))
3402	{
3403	/* MMIO pages doesn't have any readable backing. */
3404	PPGMPAGE pPage = pTlbe->pPage;
3405	if (RT_UNLIKELY(PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)))
3406	rc = VERR_PGM_PHYS_PAGE_RESERVED;
3407	else
3408	{
3409	/*
3410	* Now, just perform the locking and calculate the return address.
3411	*/
3412	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
3413	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3414	}
3415	}
3416
3417	PGM_UNLOCK(pVM);
3418	return rc;
3419	}
3420
3421
3422	/**
3423	* Requests the mapping of a guest page given by virtual address into the current context.
3424	*
3425	* This API should only be used for very short term, as it will consume
3426	* scarse resources (R0 and GC) in the mapping cache. When you're done
3427	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3428	*
3429	* This API will assume your intention is to write to the page, and will
3430	* therefore replace shared and zero pages. If you do not intend to modify
3431	* the page, use the PGMPhysGCPtr2CCPtrReadOnly() API.
3432	*
3433	* @returns VBox status code.
3434	* @retval VINF_SUCCESS on success.
3435	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3436	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3437	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3438	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3439	*
3440	* @param pVCpu The cross context virtual CPU structure.
3441	* @param GCPtr The guest physical address of the page that should be
3442	* mapped.
3443	* @param ppv Where to store the address corresponding to GCPhys.
3444	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3445	*
3446	* @remark Avoid calling this API from within critical sections (other than
3447	* the PGM one) because of the deadlock risk.
3448	* @thread EMT
3449	*/
3450	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtr, void **ppv, PPGMPAGEMAPLOCK pLock)
3451	{
3452	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3453	RTGCPHYS GCPhys;
3454	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3455	if (RT_SUCCESS(rc))
3456	rc = PGMPhysGCPhys2CCPtr(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3457	return rc;
3458	}
3459
3460
3461	/**
3462	* Requests the mapping of a guest page given by virtual address into the current context.
3463	*
3464	* This API should only be used for very short term, as it will consume
3465	* scarse resources (R0 and GC) in the mapping cache. When you're done
3466	* with the page, call PGMPhysReleasePageMappingLock() ASAP to release it.
3467	*
3468	* @returns VBox status code.
3469	* @retval VINF_SUCCESS on success.
3470	* @retval VERR_PAGE_TABLE_NOT_PRESENT if the page directory for the virtual address isn't present.
3471	* @retval VERR_PAGE_NOT_PRESENT if the page at the virtual address isn't present.
3472	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical backing.
3473	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
3474	*
3475	* @param pVCpu The cross context virtual CPU structure.
3476	* @param GCPtr The guest physical address of the page that should be
3477	* mapped.
3478	* @param ppv Where to store the address corresponding to GCPtr.
3479	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
3480	*
3481	* @remark Avoid calling this API from within critical sections (other than
3482	* the PGM one) because of the deadlock risk.
3483	* @thread EMT(pVCpu)
3484	*/
3485	VMM_INT_DECL(int) PGMPhysGCPtr2CCPtrReadOnly(PVMCPUCC pVCpu, RTGCPTR GCPtr, void const **ppv, PPGMPAGEMAPLOCK pLock)
3486	{
3487	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3488	RTGCPHYS GCPhys;
3489	int rc = PGMPhysGCPtr2GCPhys(pVCpu, GCPtr, &GCPhys);
3490	if (RT_SUCCESS(rc))
3491	rc = PGMPhysGCPhys2CCPtrReadOnly(pVCpu->CTX_SUFF(pVM), GCPhys, ppv, pLock);
3492	return rc;
3493	}
3494
3495
3496	/**
3497	* Release the mapping of a guest page.
3498	*
3499	* This is the counter part of PGMPhysGCPhys2CCPtr, PGMPhysGCPhys2CCPtrReadOnly
3500	* PGMPhysGCPtr2CCPtr and PGMPhysGCPtr2CCPtrReadOnly.
3501	*
3502	* @param pVM The cross context VM structure.
3503	* @param pLock The lock structure initialized by the mapping function.
3504	*/
3505	VMMDECL(void) PGMPhysReleasePageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3506	{
3507	# ifndef IN_RING0
3508	PPGMPAGEMAP pMap = (PPGMPAGEMAP)pLock->pvMap;
3509	# endif
3510	PPGMPAGE pPage = (PPGMPAGE)(pLock->uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3511	bool fWriteLock = (pLock->uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3512
3513	pLock->uPageAndType = 0;
3514	pLock->pvMap = NULL;
3515
3516	PGM_LOCK_VOID(pVM);
3517	if (fWriteLock)
3518	{
3519	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3520	Assert(cLocks > 0);
3521	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3522	{
3523	if (cLocks == 1)
3524	{
3525	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3526	pVM->pgm.s.cWriteLockedPages--;
3527	}
3528	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3529	}
3530
3531	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3532	{ /* probably extremely likely */ }
3533	else
3534	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3535	}
3536	else
3537	{
3538	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3539	Assert(cLocks > 0);
3540	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3541	{
3542	if (cLocks == 1)
3543	{
3544	Assert(pVM->pgm.s.cReadLockedPages > 0);
3545	pVM->pgm.s.cReadLockedPages--;
3546	}
3547	PGM_PAGE_DEC_READ_LOCKS(pPage);
3548	}
3549	}
3550
3551	# ifndef IN_RING0
3552	if (pMap)
3553	{
3554	Assert(pMap->cRefs >= 1);
3555	pMap->cRefs--;
3556	}
3557	# endif
3558	PGM_UNLOCK(pVM);
3559	}
3560
3561
3562	#ifdef IN_RING3
3563	/**
3564	* Release the mapping of multiple guest pages.
3565	*
3566	* This is the counter part to PGMR3PhysBulkGCPhys2CCPtrExternal() and
3567	* PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal().
3568	*
3569	* @param pVM The cross context VM structure.
3570	* @param cPages Number of pages to unlock.
3571	* @param paLocks Array of locks lock structure initialized by the mapping
3572	* function.
3573	*/
3574	VMMDECL(void) PGMPhysBulkReleasePageMappingLocks(PVMCC pVM, uint32_t cPages, PPGMPAGEMAPLOCK paLocks)
3575	{
3576	Assert(cPages > 0);
3577	bool const fWriteLock = (paLocks[0].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE;
3578	#ifdef VBOX_STRICT
3579	for (uint32_t i = 1; i < cPages; i++)
3580	{
3581	Assert(fWriteLock == ((paLocks[i].uPageAndType & PGMPAGEMAPLOCK_TYPE_MASK) == PGMPAGEMAPLOCK_TYPE_WRITE));
3582	AssertPtr(paLocks[i].uPageAndType);
3583	}
3584	#endif
3585
3586	PGM_LOCK_VOID(pVM);
3587	if (fWriteLock)
3588	{
3589	/*
3590	* Write locks:
3591	*/
3592	for (uint32_t i = 0; i < cPages; i++)
3593	{
3594	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3595	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
3596	Assert(cLocks > 0);
3597	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3598	{
3599	if (cLocks == 1)
3600	{
3601	Assert(pVM->pgm.s.cWriteLockedPages > 0);
3602	pVM->pgm.s.cWriteLockedPages--;
3603	}
3604	PGM_PAGE_DEC_WRITE_LOCKS(pPage);
3605	}
3606
3607	if (PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_WRITE_MONITORED)
3608	{ /* probably extremely likely */ }
3609	else
3610	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, NIL_RTGCPHYS);
3611
3612	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3613	if (pMap)
3614	{
3615	Assert(pMap->cRefs >= 1);
3616	pMap->cRefs--;
3617	}
3618
3619	/* Yield the lock: */
3620	if ((i & 1023) == 1023 && i + 1 < cPages)
3621	{
3622	PGM_UNLOCK(pVM);
3623	PGM_LOCK_VOID(pVM);
3624	}
3625	}
3626	}
3627	else
3628	{
3629	/*
3630	* Read locks:
3631	*/
3632	for (uint32_t i = 0; i < cPages; i++)
3633	{
3634	PPGMPAGE pPage = (PPGMPAGE)(paLocks[i].uPageAndType & ~PGMPAGEMAPLOCK_TYPE_MASK);
3635	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
3636	Assert(cLocks > 0);
3637	if (RT_LIKELY(cLocks > 0 && cLocks < PGM_PAGE_MAX_LOCKS))
3638	{
3639	if (cLocks == 1)
3640	{
3641	Assert(pVM->pgm.s.cReadLockedPages > 0);
3642	pVM->pgm.s.cReadLockedPages--;
3643	}
3644	PGM_PAGE_DEC_READ_LOCKS(pPage);
3645	}
3646
3647	PPGMPAGEMAP pMap = (PPGMPAGEMAP)paLocks[i].pvMap;
3648	if (pMap)
3649	{
3650	Assert(pMap->cRefs >= 1);
3651	pMap->cRefs--;
3652	}
3653
3654	/* Yield the lock: */
3655	if ((i & 1023) == 1023 && i + 1 < cPages)
3656	{
3657	PGM_UNLOCK(pVM);
3658	PGM_LOCK_VOID(pVM);
3659	}
3660	}
3661	}
3662	PGM_UNLOCK(pVM);
3663
3664	RT_BZERO(paLocks, sizeof(paLocks[0]) * cPages);
3665	}
3666	#endif /* IN_RING3 */
3667
3668
3669	/**
3670	* Release the internal mapping of a guest page.
3671	*
3672	* This is the counter part of pgmPhysGCPhys2CCPtrInternalEx and
3673	* pgmPhysGCPhys2CCPtrInternalReadOnly.
3674	*
3675	* @param pVM The cross context VM structure.
3676	* @param pLock The lock structure initialized by the mapping function.
3677	*
3678	* @remarks Caller must hold the PGM lock.
3679	*/
3680	void pgmPhysReleaseInternalPageMappingLock(PVMCC pVM, PPGMPAGEMAPLOCK pLock)
3681	{
3682	PGM_LOCK_ASSERT_OWNER(pVM);
3683	PGMPhysReleasePageMappingLock(pVM, pLock); /* lazy for now */
3684	}
3685
3686
3687	/**
3688	* Converts a GC physical address to a HC ring-3 pointer.
3689	*
3690	* @returns VINF_SUCCESS on success.
3691	* @returns VERR_PGM_PHYS_PAGE_RESERVED it it's a valid GC physical
3692	* page but has no physical backing.
3693	* @returns VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid
3694	* GC physical address.
3695	* @returns VERR_PGM_GCPHYS_RANGE_CROSSES_BOUNDARY if the range crosses
3696	* a dynamic ram chunk boundary
3697	*
3698	* @param pVM The cross context VM structure.
3699	* @param GCPhys The GC physical address to convert.
3700	* @param pR3Ptr Where to store the R3 pointer on success.
3701	*
3702	* @deprecated Avoid when possible!
3703	*/
3704	int pgmPhysGCPhys2R3Ptr(PVMCC pVM, RTGCPHYS GCPhys, PRTR3PTR pR3Ptr)
3705	{
3706	/** @todo this is kind of hacky and needs some more work. */
3707	#ifndef DEBUG_sandervl
3708	VM_ASSERT_EMT(pVM); /* no longer safe for use outside the EMT thread! */
3709	#endif
3710
3711	Log(("pgmPhysGCPhys2R3Ptr(,%RGp,): dont use this API!\n", GCPhys)); /** @todo eliminate this API! */
3712	PGM_LOCK_VOID(pVM);
3713
3714	PPGMRAMRANGE pRam;
3715	PPGMPAGE pPage;
3716	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
3717	if (RT_SUCCESS(rc))
3718	rc = pgmPhysGCPhys2CCPtrInternalDepr(pVM, pPage, GCPhys, (void **)pR3Ptr);
3719
3720	PGM_UNLOCK(pVM);
3721	Assert(rc <= VINF_SUCCESS);
3722	return rc;
3723	}
3724
3725
3726	/**
3727	* Special lockless guest physical to current context pointer convertor.
3728	*
3729	* This is mainly for the page table walking and such.
3730	*/
3731	int pgmPhysGCPhys2CCPtrLockless(PVMCPUCC pVCpu, RTGCPHYS GCPhys, void **ppv)
3732	{
3733	VMCPU_ASSERT_EMT(pVCpu);
3734
3735	/*
3736	* Get the RAM range and page structure.
3737	*/
3738	PVMCC const pVM = pVCpu->CTX_SUFF(pVM);
3739	PGMRAMRANGE volatile *pRam;
3740	PGMPAGE volatile *pPage;
3741	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3742	if (RT_SUCCESS(rc))
3743	{
3744	/*
3745	* Now, make sure it's writable (typically it is).
3746	*/
3747	if (RT_LIKELY(PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED))
3748	{ /* likely, typically */ }
3749	else
3750	{
3751	PGM_LOCK_VOID(pVM);
3752	rc = pgmPhysPageMakeWritable(pVM, (PPGMPAGE)pPage, GCPhys);
3753	if (RT_SUCCESS(rc))
3754	rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
3755	PGM_UNLOCK(pVM);
3756	if (RT_FAILURE(rc))
3757	return rc;
3758	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3 /* not returned */, ("%Rrc\n", rc));
3759	}
3760	Assert(PGM_PAGE_GET_HCPHYS(pPage) != 0 \|\| PGM_IS_IN_NEM_MODE(pVM));
3761
3762	/*
3763	* Get the mapping address.
3764	*/
3765	uint8_t *pb;
3766	#ifdef IN_RING3
3767	if (PGM_IS_IN_NEM_MODE(pVM))
3768	pb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
3769	else
3770	#endif
3771	{
3772	#ifdef IN_RING3
3773	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
3774	PPGMPAGEMAPTLBE pTlbe;
3775	rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3776	AssertLogRelRCReturn(rc, rc);
3777	pb = (uint8_t *)pTlbe->pv;
3778	RT_NOREF(pVM);
3779	# endif
3780	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
3781	PGM_LOCK(pVM);
3782	PPGMPAGEMAPTLBE pTlbe;
3783	rc = pgmPhysPageQueryTlbeWithPage(pVM, (PPGMPAGE)pPage, GCPhys, &pTlbe);
3784	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
3785	pb = (uint8_t *)pTlbe->pv;
3786	PGM_UNLOCK(pVM);
3787	RT_NOREF(pVCpu);
3788	#endif
3789	}
3790	ppv = (void )((uintptr_t)pb \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
3791	return VINF_SUCCESS;
3792	}
3793	Assert(rc <= VINF_SUCCESS);
3794	return rc;
3795	}
3796
3797
3798	/**
3799	* Converts a guest pointer to a GC physical address.
3800	*
3801	* This uses the current CR3/CR0/CR4 of the guest.
3802	*
3803	* @returns VBox status code.
3804	* @param pVCpu The cross context virtual CPU structure.
3805	* @param GCPtr The guest pointer to convert.
3806	* @param pGCPhys Where to store the GC physical address.
3807	* @thread EMT(pVCpu)
3808	*/
3809	VMMDECL(int) PGMPhysGCPtr2GCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTGCPHYS pGCPhys)
3810	{
3811	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3812	PGMPTWALK Walk;
3813	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3814	if (pGCPhys && RT_SUCCESS(rc))
3815	*pGCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK);
3816	return rc;
3817	}
3818
3819
3820	/**
3821	* Converts a guest pointer to a HC physical address.
3822	*
3823	* This uses the current CR3/CR0/CR4 of the guest.
3824	*
3825	* @returns VBox status code.
3826	* @param pVCpu The cross context virtual CPU structure.
3827	* @param GCPtr The guest pointer to convert.
3828	* @param pHCPhys Where to store the HC physical address.
3829	* @thread EMT(pVCpu)
3830	*/
3831	VMM_INT_DECL(int) PGMPhysGCPtr2HCPhys(PVMCPUCC pVCpu, RTGCPTR GCPtr, PRTHCPHYS pHCPhys)
3832	{
3833	VM_ASSERT_EMT(pVCpu->CTX_SUFF(pVM));
3834	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
3835	PGMPTWALK Walk;
3836	int rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtr, &Walk);
3837	if (RT_SUCCESS(rc))
3838	rc = PGMPhysGCPhys2HCPhys(pVM, Walk.GCPhys \| ((RTGCUINTPTR)GCPtr & GUEST_PAGE_OFFSET_MASK), pHCPhys);
3839	return rc;
3840	}
3841
3842
3843
3844	#undef LOG_GROUP
3845	#define LOG_GROUP LOG_GROUP_PGM_PHYS_ACCESS
3846
3847
3848	#if defined(IN_RING3) && defined(SOME_UNUSED_FUNCTION)
3849	/**
3850	* Cache PGMPhys memory access
3851	*
3852	* @param pVM The cross context VM structure.
3853	* @param pCache Cache structure pointer
3854	* @param GCPhys GC physical address
3855	* @param pbR3 HC pointer corresponding to physical page
3856	*
3857	* @thread EMT.
3858	*/
3859	static void pgmPhysCacheAdd(PVM pVM, PGMPHYSCACHE pCache, RTGCPHYS GCPhys, uint8_t pbR3)
3860	{
3861	uint32_t iCacheIndex;
3862
3863	Assert(VM_IS_EMT(pVM));
3864
3865	GCPhys &= ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK;
3866	pbR3 = (uint8_t *)((uintptr_t)pbR3 & ~(uintptr_t)GUEST_PAGE_OFFSET_MASK);
3867
3868	iCacheIndex = ((GCPhys >> GUEST_PAGE_SHIFT) & PGM_MAX_PHYSCACHE_ENTRIES_MASK);
3869
3870	ASMBitSet(&pCache->aEntries, iCacheIndex);
3871
3872	pCache->Entry[iCacheIndex].GCPhys = GCPhys;
3873	pCache->Entry[iCacheIndex].pbR3 = pbR3;
3874	}
3875	#endif /* IN_RING3 */
3876
3877
3878	/**
3879	* Deals with reading from a page with one or more ALL access handlers.
3880	*
3881	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
3882	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
3883	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
3884	*
3885	* @param pVM The cross context VM structure.
3886	* @param pPage The page descriptor.
3887	* @param GCPhys The physical address to start reading at.
3888	* @param pvBuf Where to put the bits we read.
3889	* @param cb How much to read - less or equal to a page.
3890	* @param enmOrigin The origin of this call.
3891	*/
3892	static VBOXSTRICTRC pgmPhysReadHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void *pvBuf, size_t cb,
3893	PGMACCESSORIGIN enmOrigin)
3894	{
3895	/*
3896	* The most frequent access here is MMIO and shadowed ROM.
3897	* The current code ASSUMES all these access handlers covers full pages!
3898	*/
3899
3900	/*
3901	* Whatever we do we need the source page, map it first.
3902	*/
3903	PGMPAGEMAPLOCK PgMpLck;
3904	const void *pvSrc = NULL;
3905	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, GCPhys, &pvSrc, &PgMpLck);
3906	/** @todo Check how this can work for MMIO pages? */
3907	if (RT_FAILURE(rc))
3908	{
3909	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
3910	GCPhys, pPage, rc));
3911	memset(pvBuf, 0xff, cb);
3912	return VINF_SUCCESS;
3913	}
3914
3915	VBOXSTRICTRC rcStrict = VINF_PGM_HANDLER_DO_DEFAULT;
3916
3917	/*
3918	* Deal with any physical handlers.
3919	*/
3920	PVMCPUCC pVCpu = VMMGetCpu(pVM);
3921	if ( PGM_PAGE_GET_HNDL_PHYS_STATE(pPage) == PGM_PAGE_HNDL_PHYS_STATE_ALL
3922	\|\| PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
3923	{
3924	PPGMPHYSHANDLER pCur;
3925	rc = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
3926	if (RT_SUCCESS(rc))
3927	{
3928	Assert(pCur && GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
3929	Assert((pCur->Key & GUEST_PAGE_OFFSET_MASK) == 0);
3930	Assert((pCur->KeyLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK);
3931	#ifndef IN_RING3
3932	if (enmOrigin != PGMACCESSORIGIN_IEM)
3933	{
3934	/* Cannot reliably handle informational status codes in this context */
3935	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3936	return VERR_PGM_PHYS_WR_HIT_HANDLER;
3937	}
3938	#endif
3939	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
3940	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler; Assert(pfnHandler);
3941	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
3942	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
3943
3944	Log5(("pgmPhysReadHandler: GCPhys=%RGp cb=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cb, pPage, R3STRING(pCur->pszDesc) ));
3945	STAM_PROFILE_START(&pCur->Stat, h);
3946	PGM_LOCK_ASSERT_OWNER(pVM);
3947
3948	/* Release the PGM lock as MMIO handlers take the IOM lock. (deadlock prevention) */
3949	PGM_UNLOCK(pVM);
3950	/* If the access origins with a device, make sure the buffer is initialized
3951	as a guard against leaking heap, stack and other info via badly written
3952	MMIO handling. @bugref{10651} */
3953	if (enmOrigin == PGMACCESSORIGIN_DEVICE)
3954	memset(pvBuf, 0xff, cb);
3955	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, (void *)pvSrc, pvBuf, cb, PGMACCESSTYPE_READ, enmOrigin, uUser);
3956	PGM_LOCK_VOID(pVM);
3957
3958	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
3959	pCur = NULL; /* might not be valid anymore. */
3960	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, false),
3961	("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys));
3962	if ( rcStrict != VINF_PGM_HANDLER_DO_DEFAULT
3963	&& !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
3964	{
3965	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3966	return rcStrict;
3967	}
3968	}
3969	else if (rc == VERR_NOT_FOUND)
3970	AssertLogRelMsgFailed(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb));
3971	else
3972	AssertLogRelMsgFailedReturn(("rc=%Rrc GCPhys=%RGp cb=%#x\n", rc, GCPhys, cb), rc);
3973	}
3974
3975	/*
3976	* Take the default action.
3977	*/
3978	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
3979	{
3980	memcpy(pvBuf, pvSrc, cb);
3981	rcStrict = VINF_SUCCESS;
3982	}
3983	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
3984	return rcStrict;
3985	}
3986
3987
3988	/**
3989	* Read physical memory.
3990	*
3991	* This API respects access handlers and MMIO. Use PGMPhysSimpleReadGCPhys() if you
3992	* want to ignore those.
3993	*
3994	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
3995	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
3996	* @retval VINF_SUCCESS in all context - read completed.
3997	*
3998	* @retval VINF_EM_OFF in RC and R0 - read completed.
3999	* @retval VINF_EM_SUSPEND in RC and R0 - read completed.
4000	* @retval VINF_EM_RESET in RC and R0 - read completed.
4001	* @retval VINF_EM_HALT in RC and R0 - read completed.
4002	* @retval VINF_SELM_SYNC_GDT in RC only - read completed.
4003	*
4004	* @retval VINF_EM_DBG_STOP in RC and R0 - read completed.
4005	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - read completed.
4006	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
4007	*
4008	* @retval VINF_IOM_R3_MMIO_READ in RC and R0.
4009	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
4010	*
4011	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
4012	*
4013	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4014	* haven't been cleared for strict status codes yet.
4015	*
4016	* @param pVM The cross context VM structure.
4017	* @param GCPhys Physical address start reading from.
4018	* @param pvBuf Where to put the read bits.
4019	* @param cbRead How many bytes to read.
4020	* @param enmOrigin The origin of this call.
4021	*/
4022	VMMDECL(VBOXSTRICTRC) PGMPhysRead(PVMCC pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
4023	{
4024	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
4025	LogFlow(("PGMPhysRead: %RGp %d\n", GCPhys, cbRead));
4026
4027	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysRead));
4028	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysReadBytes), cbRead);
4029
4030	PGM_LOCK_VOID(pVM);
4031
4032	/*
4033	* Copy loop on ram ranges.
4034	*/
4035	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4036	for (;;)
4037	{
4038	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4039
4040	/* Inside range or not? */
4041	if (pRam && GCPhys >= pRam->GCPhys)
4042	{
4043	/*
4044	* Must work our way thru this page by page.
4045	*/
4046	RTGCPHYS off = GCPhys - pRam->GCPhys;
4047	while (off < pRam->cb)
4048	{
4049	unsigned iPage = off >> GUEST_PAGE_SHIFT;
4050	PPGMPAGE pPage = &pRam->aPages[iPage];
4051	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4052	if (cb > cbRead)
4053	cb = cbRead;
4054
4055	/*
4056	* Normal page? Get the pointer to it.
4057	*/
4058	if ( !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
4059	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4060	{
4061	/*
4062	* Get the pointer to the page.
4063	*/
4064	PGMPAGEMAPLOCK PgMpLck;
4065	const void *pvSrc;
4066	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
4067	if (RT_SUCCESS(rc))
4068	{
4069	memcpy(pvBuf, pvSrc, cb);
4070	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4071	}
4072	else
4073	{
4074	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
4075	pRam->GCPhys + off, pPage, rc));
4076	memset(pvBuf, 0xff, cb);
4077	}
4078	}
4079	/*
4080	* Have ALL/MMIO access handlers.
4081	*/
4082	else
4083	{
4084	VBOXSTRICTRC rcStrict2 = pgmPhysReadHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4085	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4086	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4087	else
4088	{
4089	/* Set the remaining buffer to a known value. */
4090	memset(pvBuf, 0xff, cbRead);
4091	PGM_UNLOCK(pVM);
4092	return rcStrict2;
4093	}
4094	}
4095
4096	/* next page */
4097	if (cb >= cbRead)
4098	{
4099	PGM_UNLOCK(pVM);
4100	return rcStrict;
4101	}
4102	cbRead -= cb;
4103	off += cb;
4104	pvBuf = (char *)pvBuf + cb;
4105	} /* walk pages in ram range. */
4106
4107	GCPhys = pRam->GCPhysLast + 1;
4108	}
4109	else
4110	{
4111	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
4112
4113	/*
4114	* Unassigned address space.
4115	*/
4116	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
4117	if (cb >= cbRead)
4118	{
4119	memset(pvBuf, 0xff, cbRead);
4120	break;
4121	}
4122	memset(pvBuf, 0xff, cb);
4123
4124	cbRead -= cb;
4125	pvBuf = (char *)pvBuf + cb;
4126	GCPhys += cb;
4127	}
4128
4129	} /* Ram range walk */
4130
4131	PGM_UNLOCK(pVM);
4132	return rcStrict;
4133	}
4134
4135
4136	/**
4137	* Deals with writing to a page with one or more WRITE or ALL access handlers.
4138	*
4139	* @returns Strict VBox status code in ring-0 and raw-mode, ignorable in ring-3.
4140	* See PGM_HANDLER_PHYS_IS_VALID_STATUS and
4141	* PGM_HANDLER_VIRT_IS_VALID_STATUS for details.
4142	*
4143	* @param pVM The cross context VM structure.
4144	* @param pPage The page descriptor.
4145	* @param GCPhys The physical address to start writing at.
4146	* @param pvBuf What to write.
4147	* @param cbWrite How much to write - less or equal to a page.
4148	* @param enmOrigin The origin of this call.
4149	*/
4150	static VBOXSTRICTRC pgmPhysWriteHandler(PVMCC pVM, PPGMPAGE pPage, RTGCPHYS GCPhys, void const *pvBuf, size_t cbWrite,
4151	PGMACCESSORIGIN enmOrigin)
4152	{
4153	PGMPAGEMAPLOCK PgMpLck;
4154	void *pvDst = NULL;
4155	VBOXSTRICTRC rcStrict;
4156
4157	/*
4158	* Give priority to physical handlers (like #PF does).
4159	*
4160	* Hope for a lonely physical handler first that covers the whole write
4161	* area. This should be a pretty frequent case with MMIO and the heavy
4162	* usage of full page handlers in the page pool.
4163	*/
4164	PVMCPUCC pVCpu = VMMGetCpu(pVM);
4165	PPGMPHYSHANDLER pCur;
4166	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pCur);
4167	if (RT_SUCCESS(rcStrict))
4168	{
4169	Assert(GCPhys >= pCur->Key && GCPhys <= pCur->KeyLast);
4170	#ifndef IN_RING3
4171	if (enmOrigin != PGMACCESSORIGIN_IEM)
4172	/* Cannot reliably handle informational status codes in this context */
4173	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4174	#endif
4175	size_t cbRange = pCur->KeyLast - GCPhys + 1;
4176	if (cbRange > cbWrite)
4177	cbRange = cbWrite;
4178
4179	Assert(PGMPHYSHANDLER_GET_TYPE(pVM, pCur)->pfnHandler);
4180	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n",
4181	GCPhys, cbRange, pPage, R3STRING(pCur->pszDesc) ));
4182	if (!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
4183	rcStrict = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4184	else
4185	rcStrict = VINF_SUCCESS;
4186	if (RT_SUCCESS(rcStrict))
4187	{
4188	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pCur);
4189	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4190	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pCur->uUser
4191	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pCur->uUser);
4192	STAM_PROFILE_START(&pCur->Stat, h);
4193
4194	/* Most handlers will want to release the PGM lock for deadlock prevention
4195	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4196	dirty page trackers will want to keep it for performance reasons. */
4197	PGM_LOCK_ASSERT_OWNER(pVM);
4198	if (pCurType->fKeepPgmLock)
4199	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4200	else
4201	{
4202	PGM_UNLOCK(pVM);
4203	rcStrict = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4204	PGM_LOCK_VOID(pVM);
4205	}
4206
4207	STAM_PROFILE_STOP(&pCur->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4208	pCur = NULL; /* might not be valid anymore. */
4209	if (rcStrict == VINF_PGM_HANDLER_DO_DEFAULT)
4210	{
4211	if (pvDst)
4212	memcpy(pvDst, pvBuf, cbRange);
4213	rcStrict = VINF_SUCCESS;
4214	}
4215	else
4216	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict, true),
4217	("rcStrict=%Rrc GCPhys=%RGp pPage=%R[pgmpage] %s\n",
4218	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pCur ? R3STRING(pCur->pszDesc) : ""));
4219	}
4220	else
4221	AssertLogRelMsgFailedReturn(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4222	GCPhys, pPage, VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
4223	if (RT_LIKELY(cbRange == cbWrite) \|\| !PGM_PHYS_RW_IS_SUCCESS(rcStrict))
4224	{
4225	if (pvDst)
4226	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4227	return rcStrict;
4228	}
4229
4230	/* more fun to be had below */
4231	cbWrite -= cbRange;
4232	GCPhys += cbRange;
4233	pvBuf = (uint8_t *)pvBuf + cbRange;
4234	pvDst = (uint8_t *)pvDst + cbRange;
4235	}
4236	else if (rcStrict == VERR_NOT_FOUND) /* The handler is somewhere else in the page, deal with it below. */
4237	rcStrict = VINF_SUCCESS;
4238	else
4239	AssertMsgFailedReturn(("rcStrict=%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4240	Assert(!PGM_PAGE_IS_MMIO_OR_ALIAS(pPage)); /* MMIO handlers are all GUEST_PAGE_SIZEed! */
4241
4242	/*
4243	* Deal with all the odd ends (used to be deal with virt+phys).
4244	*/
4245	Assert(rcStrict != VINF_PGM_HANDLER_DO_DEFAULT);
4246
4247	/* We need a writable destination page. */
4248	if (!pvDst)
4249	{
4250	int rc2 = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, GCPhys, &pvDst, &PgMpLck);
4251	AssertLogRelMsgReturn(RT_SUCCESS(rc2),
4252	("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n", GCPhys, pPage, rc2),
4253	rc2);
4254	}
4255
4256	/** @todo clean up this code some more now there are no virtual handlers any
4257	* more. */
4258	/* The loop state (big + ugly). */
4259	PPGMPHYSHANDLER pPhys = NULL;
4260	uint32_t offPhys = GUEST_PAGE_SIZE;
4261	uint32_t offPhysLast = GUEST_PAGE_SIZE;
4262	bool fMorePhys = PGM_PAGE_HAS_ACTIVE_PHYSICAL_HANDLERS(pPage);
4263
4264	/* The loop. */
4265	for (;;)
4266	{
4267	if (fMorePhys && !pPhys)
4268	{
4269	rcStrict = pgmHandlerPhysicalLookup(pVM, GCPhys, &pPhys);
4270	if (RT_SUCCESS_NP(rcStrict))
4271	{
4272	offPhys = 0;
4273	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4274	}
4275	else
4276	{
4277	AssertMsgReturn(rcStrict == VERR_NOT_FOUND, ("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4278
4279	rcStrict = pVM->VMCC_CTX(pgm).s.pPhysHandlerTree->lookupMatchingOrAbove(&pVM->VMCC_CTX(pgm).s.PhysHandlerAllocator,
4280	GCPhys, &pPhys);
4281	AssertMsgReturn(RT_SUCCESS(rcStrict) \|\| rcStrict == VERR_NOT_FOUND,
4282	("%Rrc GCPhys=%RGp\n", VBOXSTRICTRC_VAL(rcStrict), GCPhys), rcStrict);
4283
4284	if ( RT_SUCCESS(rcStrict)
4285	&& pPhys->Key <= GCPhys + (cbWrite - 1))
4286	{
4287	offPhys = pPhys->Key - GCPhys;
4288	offPhysLast = pPhys->KeyLast - GCPhys; /* ASSUMES < 4GB handlers... */
4289	Assert(pPhys->KeyLast - pPhys->Key < _4G);
4290	}
4291	else
4292	{
4293	pPhys = NULL;
4294	fMorePhys = false;
4295	offPhys = offPhysLast = GUEST_PAGE_SIZE;
4296	}
4297	}
4298	}
4299
4300	/*
4301	* Handle access to space without handlers (that's easy).
4302	*/
4303	VBOXSTRICTRC rcStrict2 = VINF_PGM_HANDLER_DO_DEFAULT;
4304	uint32_t cbRange = (uint32_t)cbWrite;
4305	Assert(cbRange == cbWrite);
4306
4307	/*
4308	* Physical handler.
4309	*/
4310	if (!offPhys)
4311	{
4312	#ifndef IN_RING3
4313	if (enmOrigin != PGMACCESSORIGIN_IEM)
4314	/* Cannot reliably handle informational status codes in this context */
4315	return VERR_PGM_PHYS_WR_HIT_HANDLER;
4316	#endif
4317	if (cbRange > offPhysLast + 1)
4318	cbRange = offPhysLast + 1;
4319
4320	PCPGMPHYSHANDLERTYPEINT const pCurType = PGMPHYSHANDLER_GET_TYPE_NO_NULL(pVM, pPhys);
4321	PFNPGMPHYSHANDLER const pfnHandler = pCurType->pfnHandler;
4322	uint64_t const uUser = !pCurType->fRing0DevInsIdx ? pPhys->uUser
4323	: (uintptr_t)PDMDeviceRing0IdxToInstance(pVM, pPhys->uUser);
4324
4325	Log5(("pgmPhysWriteHandler: GCPhys=%RGp cbRange=%#x pPage=%R[pgmpage] phys %s\n", GCPhys, cbRange, pPage, R3STRING(pPhys->pszDesc) ));
4326	STAM_PROFILE_START(&pPhys->Stat, h);
4327
4328	/* Most handlers will want to release the PGM lock for deadlock prevention
4329	(esp. MMIO), though some PGM internal ones like the page pool and MMIO2
4330	dirty page trackers will want to keep it for performance reasons. */
4331	PGM_LOCK_ASSERT_OWNER(pVM);
4332	if (pCurType->fKeepPgmLock)
4333	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4334	else
4335	{
4336	PGM_UNLOCK(pVM);
4337	rcStrict2 = pfnHandler(pVM, pVCpu, GCPhys, pvDst, (void *)pvBuf, cbRange, PGMACCESSTYPE_WRITE, enmOrigin, uUser);
4338	PGM_LOCK_VOID(pVM);
4339	}
4340
4341	STAM_PROFILE_STOP(&pPhys->Stat, h); /* no locking needed, entry is unlikely reused before we get here. */
4342	pPhys = NULL; /* might not be valid anymore. */
4343	AssertLogRelMsg(PGM_HANDLER_PHYS_IS_VALID_STATUS(rcStrict2, true),
4344	("rcStrict2=%Rrc (rcStrict=%Rrc) GCPhys=%RGp pPage=%R[pgmpage] %s\n", VBOXSTRICTRC_VAL(rcStrict2),
4345	VBOXSTRICTRC_VAL(rcStrict), GCPhys, pPage, pPhys ? R3STRING(pPhys->pszDesc) : ""));
4346	}
4347
4348	/*
4349	* Execute the default action and merge the status codes.
4350	*/
4351	if (rcStrict2 == VINF_PGM_HANDLER_DO_DEFAULT)
4352	{
4353	memcpy(pvDst, pvBuf, cbRange);
4354	rcStrict2 = VINF_SUCCESS;
4355	}
4356	else if (!PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4357	{
4358	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4359	return rcStrict2;
4360	}
4361	else
4362	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4363
4364	/*
4365	* Advance if we've got more stuff to do.
4366	*/
4367	if (cbRange >= cbWrite)
4368	{
4369	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4370	return rcStrict;
4371	}
4372
4373
4374	cbWrite -= cbRange;
4375	GCPhys += cbRange;
4376	pvBuf = (uint8_t *)pvBuf + cbRange;
4377	pvDst = (uint8_t *)pvDst + cbRange;
4378
4379	offPhys -= cbRange;
4380	offPhysLast -= cbRange;
4381	}
4382	}
4383
4384
4385	/**
4386	* Write to physical memory.
4387	*
4388	* This API respects access handlers and MMIO. Use PGMPhysSimpleWriteGCPhys() if you
4389	* want to ignore those.
4390	*
4391	* @returns Strict VBox status code in raw-mode and ring-0, normal VBox status
4392	* code in ring-3. Use PGM_PHYS_RW_IS_SUCCESS to check.
4393	* @retval VINF_SUCCESS in all context - write completed.
4394	*
4395	* @retval VINF_EM_OFF in RC and R0 - write completed.
4396	* @retval VINF_EM_SUSPEND in RC and R0 - write completed.
4397	* @retval VINF_EM_RESET in RC and R0 - write completed.
4398	* @retval VINF_EM_HALT in RC and R0 - write completed.
4399	* @retval VINF_SELM_SYNC_GDT in RC only - write completed.
4400	*
4401	* @retval VINF_EM_DBG_STOP in RC and R0 - write completed.
4402	* @retval VINF_EM_DBG_BREAKPOINT in RC and R0 - write completed.
4403	* @retval VINF_EM_RAW_EMULATE_INSTR in RC and R0 only.
4404	*
4405	* @retval VINF_IOM_R3_MMIO_WRITE in RC and R0.
4406	* @retval VINF_IOM_R3_MMIO_READ_WRITE in RC and R0.
4407	* @retval VINF_IOM_R3_MMIO_COMMIT_WRITE in RC and R0.
4408	*
4409	* @retval VINF_EM_RAW_EMULATE_INSTR_GDT_FAULT in RC only - write completed.
4410	* @retval VINF_EM_RAW_EMULATE_INSTR_LDT_FAULT in RC only.
4411	* @retval VINF_EM_RAW_EMULATE_INSTR_TSS_FAULT in RC only.
4412	* @retval VINF_EM_RAW_EMULATE_INSTR_IDT_FAULT in RC only.
4413	* @retval VINF_CSAM_PENDING_ACTION in RC only.
4414	* @retval VINF_PATM_CHECK_PATCH_PAGE in RC only.
4415	*
4416	* @retval VERR_PGM_PHYS_WR_HIT_HANDLER in RC and R0 for access origins that
4417	* haven't been cleared for strict status codes yet.
4418	*
4419	*
4420	* @param pVM The cross context VM structure.
4421	* @param GCPhys Physical address to write to.
4422	* @param pvBuf What to write.
4423	* @param cbWrite How many bytes to write.
4424	* @param enmOrigin Who is calling.
4425	*/
4426	VMMDECL(VBOXSTRICTRC) PGMPhysWrite(PVMCC pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
4427	{
4428	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites, ("Calling PGMPhysWrite after pgmR3Save()! enmOrigin=%d\n", enmOrigin));
4429	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
4430	LogFlow(("PGMPhysWrite: %RGp %d\n", GCPhys, cbWrite));
4431
4432	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWrite));
4433	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysWriteBytes), cbWrite);
4434
4435	PGM_LOCK_VOID(pVM);
4436
4437	/*
4438	* Copy loop on ram ranges.
4439	*/
4440	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
4441	for (;;)
4442	{
4443	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
4444
4445	/* Inside range or not? */
4446	if (pRam && GCPhys >= pRam->GCPhys)
4447	{
4448	/*
4449	* Must work our way thru this page by page.
4450	*/
4451	RTGCPTR off = GCPhys - pRam->GCPhys;
4452	while (off < pRam->cb)
4453	{
4454	RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
4455	PPGMPAGE pPage = &pRam->aPages[iPage];
4456	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
4457	if (cb > cbWrite)
4458	cb = cbWrite;
4459
4460	/*
4461	* Normal page? Get the pointer to it.
4462	*/
4463	if ( !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
4464	&& !PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
4465	{
4466	PGMPAGEMAPLOCK PgMpLck;
4467	void *pvDst;
4468	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
4469	if (RT_SUCCESS(rc))
4470	{
4471	Assert(!PGM_PAGE_IS_BALLOONED(pPage));
4472	memcpy(pvDst, pvBuf, cb);
4473	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
4474	}
4475	/* Ignore writes to ballooned pages. */
4476	else if (!PGM_PAGE_IS_BALLOONED(pPage))
4477	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
4478	pRam->GCPhys + off, pPage, rc));
4479	}
4480	/*
4481	* Active WRITE or ALL access handlers.
4482	*/
4483	else
4484	{
4485	VBOXSTRICTRC rcStrict2 = pgmPhysWriteHandler(pVM, pPage, pRam->GCPhys + off, pvBuf, cb, enmOrigin);
4486	if (PGM_PHYS_RW_IS_SUCCESS(rcStrict2))
4487	PGM_PHYS_RW_DO_UPDATE_STRICT_RC(rcStrict, rcStrict2);
4488	else
4489	{
4490	PGM_UNLOCK(pVM);
4491	return rcStrict2;
4492	}
4493	}
4494
4495	/* next page */
4496	if (cb >= cbWrite)
4497	{
4498	PGM_UNLOCK(pVM);
4499	return rcStrict;
4500	}
4501
4502	cbWrite -= cb;
4503	off += cb;
4504	pvBuf = (const char *)pvBuf + cb;
4505	} /* walk pages in ram range */
4506
4507	GCPhys = pRam->GCPhysLast + 1;
4508	}
4509	else
4510	{
4511	/*
4512	* Unassigned address space, skip it.
4513	*/
4514	if (!pRam)
4515	break;
4516	size_t cb = pRam->GCPhys - GCPhys;
4517	if (cb >= cbWrite)
4518	break;
4519	cbWrite -= cb;
4520	pvBuf = (const char *)pvBuf + cb;
4521	GCPhys += cb;
4522	}
4523
4524	} /* Ram range walk */
4525
4526	PGM_UNLOCK(pVM);
4527	return rcStrict;
4528	}
4529
4530
4531	/**
4532	* Read from guest physical memory by GC physical address, bypassing
4533	* MMIO and access handlers.
4534	*
4535	* @returns VBox status code.
4536	* @param pVM The cross context VM structure.
4537	* @param pvDst The destination address.
4538	* @param GCPhysSrc The source address (GC physical address).
4539	* @param cb The number of bytes to read.
4540	*/
4541	VMMDECL(int) PGMPhysSimpleReadGCPhys(PVMCC pVM, void *pvDst, RTGCPHYS GCPhysSrc, size_t cb)
4542	{
4543	/*
4544	* Treat the first page as a special case.
4545	*/
4546	if (!cb)
4547	return VINF_SUCCESS;
4548
4549	/* map the 1st page */
4550	void const *pvSrc;
4551	PGMPAGEMAPLOCK Lock;
4552	int rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4553	if (RT_FAILURE(rc))
4554	return rc;
4555
4556	/* optimize for the case where access is completely within the first page. */
4557	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysSrc & GUEST_PAGE_OFFSET_MASK);
4558	if (RT_LIKELY(cb <= cbPage))
4559	{
4560	memcpy(pvDst, pvSrc, cb);
4561	PGMPhysReleasePageMappingLock(pVM, &Lock);
4562	return VINF_SUCCESS;
4563	}
4564
4565	/* copy to the end of the page. */
4566	memcpy(pvDst, pvSrc, cbPage);
4567	PGMPhysReleasePageMappingLock(pVM, &Lock);
4568	GCPhysSrc += cbPage;
4569	pvDst = (uint8_t *)pvDst + cbPage;
4570	cb -= cbPage;
4571
4572	/*
4573	* Page by page.
4574	*/
4575	for (;;)
4576	{
4577	/* map the page */
4578	rc = PGMPhysGCPhys2CCPtrReadOnly(pVM, GCPhysSrc, &pvSrc, &Lock);
4579	if (RT_FAILURE(rc))
4580	return rc;
4581
4582	/* last page? */
4583	if (cb <= GUEST_PAGE_SIZE)
4584	{
4585	memcpy(pvDst, pvSrc, cb);
4586	PGMPhysReleasePageMappingLock(pVM, &Lock);
4587	return VINF_SUCCESS;
4588	}
4589
4590	/* copy the entire page and advance */
4591	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4592	PGMPhysReleasePageMappingLock(pVM, &Lock);
4593	GCPhysSrc += GUEST_PAGE_SIZE;
4594	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4595	cb -= GUEST_PAGE_SIZE;
4596	}
4597	/* won't ever get here. */
4598	}
4599
4600
4601	/**
4602	* Write to guest physical memory referenced by GC pointer.
4603	* Write memory to GC physical address in guest physical memory.
4604	*
4605	* This will bypass MMIO and access handlers.
4606	*
4607	* @returns VBox status code.
4608	* @param pVM The cross context VM structure.
4609	* @param GCPhysDst The GC physical address of the destination.
4610	* @param pvSrc The source buffer.
4611	* @param cb The number of bytes to write.
4612	*/
4613	VMMDECL(int) PGMPhysSimpleWriteGCPhys(PVMCC pVM, RTGCPHYS GCPhysDst, const void *pvSrc, size_t cb)
4614	{
4615	LogFlow(("PGMPhysSimpleWriteGCPhys: %RGp %zu\n", GCPhysDst, cb));
4616
4617	/*
4618	* Treat the first page as a special case.
4619	*/
4620	if (!cb)
4621	return VINF_SUCCESS;
4622
4623	/* map the 1st page */
4624	void *pvDst;
4625	PGMPAGEMAPLOCK Lock;
4626	int rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4627	if (RT_FAILURE(rc))
4628	return rc;
4629
4630	/* optimize for the case where access is completely within the first page. */
4631	size_t cbPage = GUEST_PAGE_SIZE - (GCPhysDst & GUEST_PAGE_OFFSET_MASK);
4632	if (RT_LIKELY(cb <= cbPage))
4633	{
4634	memcpy(pvDst, pvSrc, cb);
4635	PGMPhysReleasePageMappingLock(pVM, &Lock);
4636	return VINF_SUCCESS;
4637	}
4638
4639	/* copy to the end of the page. */
4640	memcpy(pvDst, pvSrc, cbPage);
4641	PGMPhysReleasePageMappingLock(pVM, &Lock);
4642	GCPhysDst += cbPage;
4643	pvSrc = (const uint8_t *)pvSrc + cbPage;
4644	cb -= cbPage;
4645
4646	/*
4647	* Page by page.
4648	*/
4649	for (;;)
4650	{
4651	/* map the page */
4652	rc = PGMPhysGCPhys2CCPtr(pVM, GCPhysDst, &pvDst, &Lock);
4653	if (RT_FAILURE(rc))
4654	return rc;
4655
4656	/* last page? */
4657	if (cb <= GUEST_PAGE_SIZE)
4658	{
4659	memcpy(pvDst, pvSrc, cb);
4660	PGMPhysReleasePageMappingLock(pVM, &Lock);
4661	return VINF_SUCCESS;
4662	}
4663
4664	/* copy the entire page and advance */
4665	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4666	PGMPhysReleasePageMappingLock(pVM, &Lock);
4667	GCPhysDst += GUEST_PAGE_SIZE;
4668	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4669	cb -= GUEST_PAGE_SIZE;
4670	}
4671	/* won't ever get here. */
4672	}
4673
4674
4675	/**
4676	* Read from guest physical memory referenced by GC pointer.
4677	*
4678	* This function uses the current CR3/CR0/CR4 of the guest and will
4679	* bypass access handlers and not set any accessed bits.
4680	*
4681	* @returns VBox status code.
4682	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4683	* @param pvDst The destination address.
4684	* @param GCPtrSrc The source address (GC pointer).
4685	* @param cb The number of bytes to read.
4686	*/
4687	VMMDECL(int) PGMPhysSimpleReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb)
4688	{
4689	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4690	/** @todo fix the macro / state handling: VMCPU_ASSERT_EMT_OR_GURU(pVCpu); */
4691
4692	/*
4693	* Treat the first page as a special case.
4694	*/
4695	if (!cb)
4696	return VINF_SUCCESS;
4697
4698	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleRead));
4699	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleReadBytes), cb);
4700
4701	/* Take the PGM lock here, because many called functions take the lock for a very short period. That's counter-productive
4702	* when many VCPUs are fighting for the lock.
4703	*/
4704	PGM_LOCK_VOID(pVM);
4705
4706	/* map the 1st page */
4707	void const *pvSrc;
4708	PGMPAGEMAPLOCK Lock;
4709	int rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4710	if (RT_FAILURE(rc))
4711	{
4712	PGM_UNLOCK(pVM);
4713	return rc;
4714	}
4715
4716	/* optimize for the case where access is completely within the first page. */
4717	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4718	if (RT_LIKELY(cb <= cbPage))
4719	{
4720	memcpy(pvDst, pvSrc, cb);
4721	PGMPhysReleasePageMappingLock(pVM, &Lock);
4722	PGM_UNLOCK(pVM);
4723	return VINF_SUCCESS;
4724	}
4725
4726	/* copy to the end of the page. */
4727	memcpy(pvDst, pvSrc, cbPage);
4728	PGMPhysReleasePageMappingLock(pVM, &Lock);
4729	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + cbPage);
4730	pvDst = (uint8_t *)pvDst + cbPage;
4731	cb -= cbPage;
4732
4733	/*
4734	* Page by page.
4735	*/
4736	for (;;)
4737	{
4738	/* map the page */
4739	rc = PGMPhysGCPtr2CCPtrReadOnly(pVCpu, GCPtrSrc, &pvSrc, &Lock);
4740	if (RT_FAILURE(rc))
4741	{
4742	PGM_UNLOCK(pVM);
4743	return rc;
4744	}
4745
4746	/* last page? */
4747	if (cb <= GUEST_PAGE_SIZE)
4748	{
4749	memcpy(pvDst, pvSrc, cb);
4750	PGMPhysReleasePageMappingLock(pVM, &Lock);
4751	PGM_UNLOCK(pVM);
4752	return VINF_SUCCESS;
4753	}
4754
4755	/* copy the entire page and advance */
4756	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4757	PGMPhysReleasePageMappingLock(pVM, &Lock);
4758	GCPtrSrc = (RTGCPTR)((RTGCUINTPTR)GCPtrSrc + GUEST_PAGE_SIZE);
4759	pvDst = (uint8_t *)pvDst + GUEST_PAGE_SIZE;
4760	cb -= GUEST_PAGE_SIZE;
4761	}
4762	/* won't ever get here. */
4763	}
4764
4765
4766	/**
4767	* Write to guest physical memory referenced by GC pointer.
4768	*
4769	* This function uses the current CR3/CR0/CR4 of the guest and will
4770	* bypass access handlers and not set dirty or accessed bits.
4771	*
4772	* @returns VBox status code.
4773	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4774	* @param GCPtrDst The destination address (GC pointer).
4775	* @param pvSrc The source address.
4776	* @param cb The number of bytes to write.
4777	*/
4778	VMMDECL(int) PGMPhysSimpleWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4779	{
4780	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4781	VMCPU_ASSERT_EMT(pVCpu);
4782
4783	/*
4784	* Treat the first page as a special case.
4785	*/
4786	if (!cb)
4787	return VINF_SUCCESS;
4788
4789	STAM_COUNTER_INC(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWrite));
4790	STAM_COUNTER_ADD(&pVM->pgm.s.Stats.CTX_MID_Z(Stat,PhysSimpleWriteBytes), cb);
4791
4792	/* map the 1st page */
4793	void *pvDst;
4794	PGMPAGEMAPLOCK Lock;
4795	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4796	if (RT_FAILURE(rc))
4797	return rc;
4798
4799	/* optimize for the case where access is completely within the first page. */
4800	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4801	if (RT_LIKELY(cb <= cbPage))
4802	{
4803	memcpy(pvDst, pvSrc, cb);
4804	PGMPhysReleasePageMappingLock(pVM, &Lock);
4805	return VINF_SUCCESS;
4806	}
4807
4808	/* copy to the end of the page. */
4809	memcpy(pvDst, pvSrc, cbPage);
4810	PGMPhysReleasePageMappingLock(pVM, &Lock);
4811	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4812	pvSrc = (const uint8_t *)pvSrc + cbPage;
4813	cb -= cbPage;
4814
4815	/*
4816	* Page by page.
4817	*/
4818	for (;;)
4819	{
4820	/* map the page */
4821	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4822	if (RT_FAILURE(rc))
4823	return rc;
4824
4825	/* last page? */
4826	if (cb <= GUEST_PAGE_SIZE)
4827	{
4828	memcpy(pvDst, pvSrc, cb);
4829	PGMPhysReleasePageMappingLock(pVM, &Lock);
4830	return VINF_SUCCESS;
4831	}
4832
4833	/* copy the entire page and advance */
4834	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4835	PGMPhysReleasePageMappingLock(pVM, &Lock);
4836	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4837	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4838	cb -= GUEST_PAGE_SIZE;
4839	}
4840	/* won't ever get here. */
4841	}
4842
4843
4844	/**
4845	* Write to guest physical memory referenced by GC pointer and update the PTE.
4846	*
4847	* This function uses the current CR3/CR0/CR4 of the guest and will
4848	* bypass access handlers but will set any dirty and accessed bits in the PTE.
4849	*
4850	* If you don't want to set the dirty bit, use PGMPhysSimpleWriteGCPtr().
4851	*
4852	* @returns VBox status code.
4853	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4854	* @param GCPtrDst The destination address (GC pointer).
4855	* @param pvSrc The source address.
4856	* @param cb The number of bytes to write.
4857	*/
4858	VMMDECL(int) PGMPhysSimpleDirtyWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb)
4859	{
4860	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4861	VMCPU_ASSERT_EMT(pVCpu);
4862
4863	/*
4864	* Treat the first page as a special case.
4865	* Btw. this is the same code as in PGMPhyssimpleWriteGCPtr excep for the PGMGstModifyPage.
4866	*/
4867	if (!cb)
4868	return VINF_SUCCESS;
4869
4870	/* map the 1st page */
4871	void *pvDst;
4872	PGMPAGEMAPLOCK Lock;
4873	int rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4874	if (RT_FAILURE(rc))
4875	return rc;
4876
4877	/* optimize for the case where access is completely within the first page. */
4878	size_t cbPage = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
4879	if (RT_LIKELY(cb <= cbPage))
4880	{
4881	memcpy(pvDst, pvSrc, cb);
4882	PGMPhysReleasePageMappingLock(pVM, &Lock);
4883	#ifdef VBOX_VMM_TARGET_X86
4884	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4885	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4886	# error "misconfig"
4887	#endif
4888	return VINF_SUCCESS;
4889	}
4890
4891	/* copy to the end of the page. */
4892	memcpy(pvDst, pvSrc, cbPage);
4893	PGMPhysReleasePageMappingLock(pVM, &Lock);
4894	#ifdef VBOX_VMM_TARGET_X86
4895	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4896	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4897	# error "misconfig"
4898	#endif
4899	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + cbPage);
4900	pvSrc = (const uint8_t *)pvSrc + cbPage;
4901	cb -= cbPage;
4902
4903	/*
4904	* Page by page.
4905	*/
4906	for (;;)
4907	{
4908	/* map the page */
4909	rc = PGMPhysGCPtr2CCPtr(pVCpu, GCPtrDst, &pvDst, &Lock);
4910	if (RT_FAILURE(rc))
4911	return rc;
4912
4913	/* last page? */
4914	if (cb <= GUEST_PAGE_SIZE)
4915	{
4916	memcpy(pvDst, pvSrc, cb);
4917	PGMPhysReleasePageMappingLock(pVM, &Lock);
4918	#ifdef VBOX_VMM_TARGET_X86
4919	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4920	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4921	# error "misconfig"
4922	#endif
4923	return VINF_SUCCESS;
4924	}
4925
4926	/* copy the entire page and advance */
4927	memcpy(pvDst, pvSrc, GUEST_PAGE_SIZE);
4928	PGMPhysReleasePageMappingLock(pVM, &Lock);
4929	#ifdef VBOX_VMM_TARGET_X86
4930	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D)); AssertRC(rc);
4931	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4932	# error "misconfig"
4933	#endif
4934	GCPtrDst = (RTGCPTR)((RTGCUINTPTR)GCPtrDst + GUEST_PAGE_SIZE);
4935	pvSrc = (const uint8_t *)pvSrc + GUEST_PAGE_SIZE;
4936	cb -= GUEST_PAGE_SIZE;
4937	}
4938	/* won't ever get here. */
4939	}
4940
4941
4942	/**
4943	* Read from guest physical memory referenced by GC pointer.
4944	*
4945	* This function uses the current CR3/CR0/CR4 of the guest and will
4946	* respect access handlers and set accessed bits.
4947	*
4948	* @returns Strict VBox status, see PGMPhysRead for details.
4949	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
4950	* specified virtual address.
4951	*
4952	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
4953	* @param pvDst The destination address.
4954	* @param GCPtrSrc The source address (GC pointer).
4955	* @param cb The number of bytes to read.
4956	* @param enmOrigin Who is calling.
4957	* @thread EMT(pVCpu)
4958	*/
4959	VMMDECL(VBOXSTRICTRC) PGMPhysReadGCPtr(PVMCPUCC pVCpu, void *pvDst, RTGCPTR GCPtrSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
4960	{
4961	int rc;
4962	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
4963	VMCPU_ASSERT_EMT(pVCpu);
4964
4965	/*
4966	* Anything to do?
4967	*/
4968	if (!cb)
4969	return VINF_SUCCESS;
4970
4971	LogFlow(("PGMPhysReadGCPtr: %RGv %zu\n", GCPtrSrc, cb));
4972
4973	/*
4974	* Optimize reads within a single page.
4975	*/
4976	if (((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
4977	{
4978	/* Convert virtual to physical address + flags */
4979	PGMPTWALK Walk;
4980	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
4981	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
4982	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
4983
4984	#ifdef VBOX_VMM_TARGET_X86
4985	/* mark the guest page as accessed. */
4986	if (!(Walk.fEffective & X86_PTE_A))
4987	{
4988	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
4989	AssertRC(rc);
4990	}
4991	#elif !defined(VBOX_VMM_TARGET_ARMV8)
4992	# error "misconfig"
4993	#endif
4994	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
4995	}
4996
4997	/*
4998	* Page by page.
4999	*/
5000	for (;;)
5001	{
5002	/* Convert virtual to physical address + flags */
5003	PGMPTWALK Walk;
5004	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrSrc, &Walk);
5005	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrSrc), rc);
5006	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
5007
5008	#ifdef VBOX_VMM_TARGET_X86
5009	/* mark the guest page as accessed. */
5010	if (!(Walk.fEffective & X86_PTE_A))
5011	{
5012	rc = PGMGstModifyPage(pVCpu, GCPtrSrc, 1, X86_PTE_A, ~(uint64_t)(X86_PTE_A));
5013	AssertRC(rc);
5014	}
5015	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5016	# error "misconfig"
5017	#endif
5018
5019	/* copy */
5020	size_t cbRead = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrSrc & GUEST_PAGE_OFFSET_MASK);
5021	if (cbRead < cb)
5022	{
5023	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, GCPhys, pvDst, cbRead, enmOrigin);
5024	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5025	{ /* likely */ }
5026	else
5027	return rcStrict;
5028	}
5029	else /* Last page (cbRead is GUEST_PAGE_SIZE, we only need cb!) */
5030	return PGMPhysRead(pVM, GCPhys, pvDst, cb, enmOrigin);
5031
5032	/* next */
5033	Assert(cb > cbRead);
5034	cb -= cbRead;
5035	pvDst = (uint8_t *)pvDst + cbRead;
5036	GCPtrSrc += cbRead;
5037	}
5038	}
5039
5040
5041	/**
5042	* Write to guest physical memory referenced by GC pointer.
5043	*
5044	* This function uses the current CR3/CR0/CR4 of the guest and will
5045	* respect access handlers and set dirty and accessed bits.
5046	*
5047	* @returns Strict VBox status, see PGMPhysWrite for details.
5048	* @retval VERR_PAGE_TABLE_NOT_PRESENT if there is no page mapped at the
5049	* specified virtual address.
5050	*
5051	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
5052	* @param GCPtrDst The destination address (GC pointer).
5053	* @param pvSrc The source address.
5054	* @param cb The number of bytes to write.
5055	* @param enmOrigin Who is calling.
5056	*/
5057	VMMDECL(VBOXSTRICTRC) PGMPhysWriteGCPtr(PVMCPUCC pVCpu, RTGCPTR GCPtrDst, const void *pvSrc, size_t cb, PGMACCESSORIGIN enmOrigin)
5058	{
5059	int rc;
5060	PVMCC pVM = pVCpu->CTX_SUFF(pVM);
5061	VMCPU_ASSERT_EMT(pVCpu);
5062
5063	/*
5064	* Anything to do?
5065	*/
5066	if (!cb)
5067	return VINF_SUCCESS;
5068
5069	LogFlow(("PGMPhysWriteGCPtr: %RGv %zu\n", GCPtrDst, cb));
5070
5071	/*
5072	* Optimize writes within a single page.
5073	*/
5074	if (((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK) + cb <= GUEST_PAGE_SIZE)
5075	{
5076	/* Convert virtual to physical address + flags */
5077	PGMPTWALK Walk;
5078	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5079	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5080	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5081
5082	/* Mention when we ignore X86_PTE_RW... */
5083	if (!(Walk.fEffective & X86_PTE_RW))
5084	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5085
5086	#ifdef VBOX_VMM_TARGET_X86
5087	/* Mark the guest page as accessed and dirty if necessary. */
5088	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5089	{
5090	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5091	AssertRC(rc);
5092	}
5093	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5094	# error "misconfig"
5095	#endif
5096
5097	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5098	}
5099
5100	/*
5101	* Page by page.
5102	*/
5103	for (;;)
5104	{
5105	/* Convert virtual to physical address + flags */
5106	PGMPTWALK Walk;
5107	rc = PGMGstGetPage(pVCpu, (RTGCUINTPTR)GCPtrDst, &Walk);
5108	AssertMsgRCReturn(rc, ("GetPage failed with %Rrc for %RGv\n", rc, GCPtrDst), rc);
5109	RTGCPHYS const GCPhys = Walk.GCPhys \| ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5110
5111	/* Mention when we ignore X86_PTE_RW... */
5112	if (!(Walk.fEffective & X86_PTE_RW))
5113	Log(("PGMPhysWriteGCPtr: Writing to RO page %RGv %#x\n", GCPtrDst, cb));
5114
5115	#ifdef VBOX_VMM_TARGET_X86
5116	/* Mark the guest page as accessed and dirty if necessary. */
5117	if ((Walk.fEffective & (X86_PTE_A \| X86_PTE_D)) != (X86_PTE_A \| X86_PTE_D))
5118	{
5119	rc = PGMGstModifyPage(pVCpu, GCPtrDst, 1, X86_PTE_A \| X86_PTE_D, ~(uint64_t)(X86_PTE_A \| X86_PTE_D));
5120	AssertRC(rc);
5121	}
5122	#elif !defined(VBOX_VMM_TARGET_ARMV8)
5123	# error "misconfig"
5124	#endif
5125
5126	/* copy */
5127	size_t cbWrite = GUEST_PAGE_SIZE - ((RTGCUINTPTR)GCPtrDst & GUEST_PAGE_OFFSET_MASK);
5128	if (cbWrite < cb)
5129	{
5130	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, GCPhys, pvSrc, cbWrite, enmOrigin);
5131	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
5132	{ /* likely */ }
5133	else
5134	return rcStrict;
5135	}
5136	else /* Last page (cbWrite is GUEST_PAGE_SIZE, we only need cb!) */
5137	return PGMPhysWrite(pVM, GCPhys, pvSrc, cb, enmOrigin);
5138
5139	/* next */
5140	Assert(cb > cbWrite);
5141	cb -= cbWrite;
5142	pvSrc = (uint8_t *)pvSrc + cbWrite;
5143	GCPtrDst += cbWrite;
5144	}
5145	}
5146
5147
5148	/**
5149	* Return the page type of the specified physical address.
5150	*
5151	* @returns The page type.
5152	* @param pVM The cross context VM structure.
5153	* @param GCPhys Guest physical address
5154	*/
5155	VMM_INT_DECL(PGMPAGETYPE) PGMPhysGetPageType(PVMCC pVM, RTGCPHYS GCPhys)
5156	{
5157	PGM_LOCK_VOID(pVM);
5158	PPGMPAGE pPage = pgmPhysGetPage(pVM, GCPhys);
5159	PGMPAGETYPE enmPgType = pPage ? (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage) : PGMPAGETYPE_INVALID;
5160	PGM_UNLOCK(pVM);
5161
5162	return enmPgType;
5163	}
5164
5165
5166	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5167	DECL_FORCE_INLINE(int)
5168	pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uint64_t uTlbPhysRev, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb,
5169	RTGCPHYS GCPhys, PCPGMPAGE pPageCopy)
5170	{
5171	*pfTlb \|= uTlbPhysRev
5172	\| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3;
5173	*ppb = NULL;
5174	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=NULL pfTlb=%#RX64 PageCopy=%R[pgmpage] NO\n", GCPhys,
5175	uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ \| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3, pPageCopy));
5176	RT_NOREF(GCPhys, pPageCopy);
5177	return VINF_SUCCESS;
5178	}
5179
5180
5181	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5182	DECL_FORCE_INLINE(int)
5183	pgmPhyIemGCphys2PtrNoLockReturnReadOnly(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5184	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5185	{
5186	RT_NOREF(GCPhys);
5187	if (!PGM_PAGE_IS_CODE_PAGE(pPageCopy))
5188	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE;
5189	else
5190	*pfTlb \|= uTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_CODE_PAGE;
5191
5192	#ifdef IN_RING3
5193	if (PGM_IS_IN_NEM_MODE(pVM))
5194	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5195	else
5196	#endif
5197	{
5198	#ifdef IN_RING3
5199	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5200	PPGMPAGEMAPTLBE pTlbe;
5201	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5202	AssertLogRelRCReturn(rc, rc);
5203	ppb = (uint8_t )pTlbe->pv;
5204	RT_NOREF(pVM);
5205	# endif
5206	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5207	PGM_LOCK(pVM);
5208	PPGMPAGEMAPTLBE pTlbe;
5209	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5210	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5211	ppb = (uint8_t )pTlbe->pv;
5212	PGM_UNLOCK(pVM);
5213	RT_NOREF(pVCpu);
5214	#endif
5215	}
5216	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RO\n", GCPhys, ppb, pfTlb, pPageCopy));
5217	RT_NOREF(pRam, pVM, pVCpu);
5218	return VINF_SUCCESS;
5219	}
5220
5221
5222	/** Helper for PGMPhysIemGCPhys2PtrNoLock. */
5223	DECL_FORCE_INLINE(int)
5224	pgmPhyIemGCphys2PtrNoLockReturnReadWrite(PVMCC pVM, PVMCPUCC pVCpu, uint64_t uTlbPhysRev, RTGCPHYS GCPhys, PCPGMPAGE pPageCopy,
5225	PPGMRAMRANGE pRam, PPGMPAGE pPage, R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5226	{
5227	Assert(!PGM_PAGE_IS_CODE_PAGE(pPageCopy));
5228	RT_NOREF(pPageCopy, GCPhys);
5229	*pfTlb \|= uTlbPhysRev;
5230
5231	#ifdef IN_RING3
5232	if (PGM_IS_IN_NEM_MODE(pVM))
5233	*ppb = &pRam->pbR3[(RTGCPHYS)(uintptr_t)(pPage - &pRam->aPages[0]) << GUEST_PAGE_SHIFT];
5234	else
5235	#endif
5236	{
5237	#ifdef IN_RING3
5238	# ifndef VBOX_WITH_ONLY_PGM_NEM_MODE
5239	PPGMPAGEMAPTLBE pTlbe;
5240	int rc = pgmPhysPageQueryLocklessTlbeWithPage(pVCpu, pPage, GCPhys, &pTlbe);
5241	AssertLogRelRCReturn(rc, rc);
5242	ppb = (uint8_t )pTlbe->pv;
5243	RT_NOREF(pVM);
5244	# endif
5245	#else /** @todo a safe lockless page TLB in ring-0 needs the to ensure it gets the right invalidations. later. */
5246	PGM_LOCK(pVM);
5247	PPGMPAGEMAPTLBE pTlbe;
5248	int rc = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5249	AssertLogRelRCReturnStmt(rc, PGM_UNLOCK(pVM), rc);
5250	ppb = (uint8_t )pTlbe->pv;
5251	PGM_UNLOCK(pVM);
5252	RT_NOREF(pVCpu);
5253	#endif
5254	}
5255	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 PageCopy=%R[pgmpage] RW\n", GCPhys, ppb, pfTlb, pPageCopy));
5256	RT_NOREF(pRam, pVM, pVCpu);
5257	return VINF_SUCCESS;
5258	}
5259
5260
5261	/**
5262	* Converts a GC physical address to a HC ring-3 pointer, with some
5263	* additional checks.
5264	*
5265	* @returns VBox status code (no informational statuses).
5266	*
5267	* @param pVM The cross context VM structure.
5268	* @param pVCpu The cross context virtual CPU structure of the
5269	* calling EMT.
5270	* @param GCPhys The GC physical address to convert. This API mask
5271	* the A20 line when necessary.
5272	* @param puTlbPhysRev Where to read the physical TLB revision. Needs to
5273	* be done while holding the PGM lock.
5274	* @param ppb Where to store the pointer corresponding to GCPhys
5275	* on success.
5276	* @param pfTlb The TLB flags and revision. We only add stuff.
5277	*
5278	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr and
5279	* PGMPhysIemGCPhys2Ptr.
5280	*
5281	* @thread EMT(pVCpu).
5282	*/
5283	VMM_INT_DECL(int) PGMPhysIemGCPhys2PtrNoLock(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, uint64_t const volatile *puTlbPhysRev,
5284	R3R0PTRTYPE(uint8_t ) ppb, uint64_t *pfTlb)
5285	{
5286	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5287	Assert(!(GCPhys & X86_PAGE_OFFSET_MASK));
5288
5289	PGMRAMRANGE volatile *pRam;
5290	PGMPAGE volatile *pPage;
5291	int rc = pgmPhysGetPageAndRangeExLockless(pVM, pVCpu, GCPhys, &pPage, &pRam);
5292	if (RT_SUCCESS(rc))
5293	{
5294	/*
5295	* Wrt to update races, we will try to pretend we beat the update we're
5296	* racing. We do this by sampling the physical TLB revision first, so
5297	* that the TLB entry / whatever purpose the caller has with the info
5298	* will become invalid immediately if it's updated.
5299	*
5300	* This means the caller will (probably) make use of the returned info
5301	* only once and then requery it the next time it is use, getting the
5302	* updated info. This would then be just as if the first query got the
5303	* PGM lock before the updater.
5304	*/
5305	/** @todo make PGMPAGE updates more atomic, possibly flagging complex
5306	* updates by adding a u1UpdateInProgress field (or revision).
5307	* This would be especially important when updating the page ID... */
5308	uint64_t uTlbPhysRev = *puTlbPhysRev;
5309	PGMPAGE PageCopy = { { pPage->au64[0], pPage->au64[1] } };
5310	if ( uTlbPhysRev == *puTlbPhysRev
5311	&& PageCopy.au64[0] == pPage->au64[0]
5312	&& PageCopy.au64[1] == pPage->au64[1])
5313	ASMCompilerBarrier(); /* likely */
5314	else
5315	{
5316	PGM_LOCK_VOID(pVM);
5317	uTlbPhysRev = *puTlbPhysRev;
5318	PageCopy.au64[0] = pPage->au64[0];
5319	PageCopy.au64[1] = pPage->au64[1];
5320	PGM_UNLOCK(pVM);
5321	}
5322
5323	/*
5324	* Try optimize for the regular case first: Writable RAM.
5325	*/
5326	switch (PGM_PAGE_GET_HNDL_PHYS_STATE(&PageCopy))
5327	{
5328	case PGM_PAGE_HNDL_PHYS_STATE_DISABLED:
5329	if (!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy))
5330	{ /* likely */ }
5331	else
5332	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5333	RT_FALL_THRU();
5334	case PGM_PAGE_HNDL_PHYS_STATE_NONE:
5335	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5336	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5337	{
5338	case PGM_PAGE_STATE_ALLOCATED:
5339	return pgmPhyIemGCphys2PtrNoLockReturnReadWrite(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5340	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5341
5342	case PGM_PAGE_STATE_ZERO:
5343	case PGM_PAGE_STATE_WRITE_MONITORED:
5344	case PGM_PAGE_STATE_SHARED:
5345	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5346	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5347
5348	default: AssertFailed(); RT_FALL_THROUGH();
5349	case PGM_PAGE_STATE_BALLOONED:
5350	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5351	}
5352	break;
5353
5354	case PGM_PAGE_HNDL_PHYS_STATE_WRITE:
5355	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5356	switch (PGM_PAGE_GET_STATE_NA(&PageCopy))
5357	{
5358	case PGM_PAGE_STATE_ALLOCATED:
5359	Assert(!PGM_PAGE_IS_CODE_PAGE(&PageCopy));
5360	RT_FALL_THRU();
5361	case PGM_PAGE_STATE_ZERO:
5362	case PGM_PAGE_STATE_WRITE_MONITORED:
5363	case PGM_PAGE_STATE_SHARED:
5364	return pgmPhyIemGCphys2PtrNoLockReturnReadOnly(pVM, pVCpu, uTlbPhysRev, GCPhys, &PageCopy,
5365	(PPGMRAMRANGE)pRam, (PPGMPAGE)pPage, ppb, pfTlb);
5366
5367	default: AssertFailed(); RT_FALL_THROUGH();
5368	case PGM_PAGE_STATE_BALLOONED:
5369	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5370	}
5371	break;
5372
5373	case PGM_PAGE_HNDL_PHYS_STATE_ALL:
5374	Assert(!PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(&PageCopy));
5375	return pgmPhyIemGCphys2PtrNoLockReturnNoNothing(uTlbPhysRev, ppb, pfTlb, GCPhys, &PageCopy);
5376	}
5377	}
5378	else
5379	{
5380	pfTlb \|= puTlbPhysRev \| PGMIEMGCPHYS2PTR_F_NO_WRITE \| PGMIEMGCPHYS2PTR_F_NO_READ
5381	\| PGMIEMGCPHYS2PTR_F_NO_MAPPINGR3 \| PGMIEMGCPHYS2PTR_F_UNASSIGNED;
5382	*ppb = NULL;
5383	Log6(("PGMPhysIemGCPhys2PtrNoLock: GCPhys=%RGp ppb=%p pfTlb=%#RX64 (rc=%Rrc)\n", GCPhys, ppb, pfTlb, rc));
5384	}
5385
5386	return VINF_SUCCESS;
5387	}
5388
5389
5390	/**
5391	* Converts a GC physical address to a HC ring-3 pointer, with some
5392	* additional checks.
5393	*
5394	* @returns VBox status code (no informational statuses).
5395	* @retval VINF_SUCCESS on success.
5396	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5397	* access handler of some kind.
5398	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5399	* accesses or is odd in any way.
5400	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5401	*
5402	* @param pVM The cross context VM structure.
5403	* @param pVCpu The cross context virtual CPU structure of the
5404	* calling EMT.
5405	* @param GCPhys The GC physical address to convert. This API mask
5406	* the A20 line when necessary.
5407	* @param fWritable Whether write access is required.
5408	* @param fByPassHandlers Whether to bypass access handlers.
5409	* @param ppv Where to store the pointer corresponding to GCPhys
5410	* on success.
5411	* @param pLock
5412	*
5413	* @remarks This is more or a less a copy of PGMR3PhysTlbGCPhys2Ptr.
5414	* @thread EMT(pVCpu).
5415	*/
5416	VMM_INT_DECL(int) PGMPhysIemGCPhys2Ptr(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers,
5417	void **ppv, PPGMPAGEMAPLOCK pLock)
5418	{
5419	PGM_A20_APPLY_TO_VAR(pVCpu, GCPhys);
5420	RT_NOREF(pVCpu);
5421
5422	PGM_LOCK_VOID(pVM);
5423
5424	PPGMRAMRANGE pRam;
5425	PPGMPAGE pPage;
5426	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5427	if (RT_SUCCESS(rc))
5428	{
5429	if (PGM_PAGE_IS_BALLOONED(pPage))
5430	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5431	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5432	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5433	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5434	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5435	rc = VINF_SUCCESS;
5436	else
5437	{
5438	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5439	{
5440	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5441	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5442	}
5443	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5444	{
5445	Assert(!fByPassHandlers);
5446	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5447	}
5448	}
5449	if (RT_SUCCESS(rc))
5450	{
5451	int rc2;
5452
5453	/* Make sure what we return is writable. */
5454	if (fWritable)
5455	switch (PGM_PAGE_GET_STATE(pPage))
5456	{
5457	case PGM_PAGE_STATE_ALLOCATED:
5458	break;
5459	case PGM_PAGE_STATE_BALLOONED:
5460	AssertFailed();
5461	break;
5462	case PGM_PAGE_STATE_ZERO:
5463	case PGM_PAGE_STATE_SHARED:
5464	case PGM_PAGE_STATE_WRITE_MONITORED:
5465	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
5466	AssertLogRelRCReturn(rc2, rc2);
5467	break;
5468	}
5469
5470	/* Get a ring-3 mapping of the address. */
5471	PPGMPAGEMAPTLBE pTlbe;
5472	rc2 = pgmPhysPageQueryTlbeWithPage(pVM, pPage, GCPhys, &pTlbe);
5473	AssertLogRelRCReturn(rc2, rc2);
5474
5475	/* Lock it and calculate the address. */
5476	if (fWritable)
5477	pgmPhysPageMapLockForWriting(pVM, pPage, pTlbe, pLock);
5478	else
5479	pgmPhysPageMapLockForReading(pVM, pPage, pTlbe, pLock);
5480	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
5481
5482	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
5483	}
5484	else
5485	Log6(("PGMPhysIemGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
5486
5487	/* else: handler catching all access, no pointer returned. */
5488	}
5489	else
5490	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
5491
5492	PGM_UNLOCK(pVM);
5493	return rc;
5494	}
5495
5496
5497	/**
5498	* Checks if the give GCPhys page requires special handling for the given access
5499	* because it's MMIO or otherwise monitored.
5500	*
5501	* @returns VBox status code (no informational statuses).
5502	* @retval VINF_SUCCESS on success.
5503	* @retval VERR_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
5504	* access handler of some kind.
5505	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
5506	* accesses or is odd in any way.
5507	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
5508	*
5509	* @param pVM The cross context VM structure.
5510	* @param GCPhys The GC physical address to convert. Since this is
5511	* only used for filling the REM TLB, the A20 mask must
5512	* be applied before calling this API.
5513	* @param fWritable Whether write access is required.
5514	* @param fByPassHandlers Whether to bypass access handlers.
5515	*
5516	* @remarks This is a watered down version PGMPhysIemGCPhys2Ptr and really just
5517	* a stop gap thing that should be removed once there is a better TLB
5518	* for virtual address accesses.
5519	*/
5520	VMM_INT_DECL(int) PGMPhysIemQueryAccess(PVMCC pVM, RTGCPHYS GCPhys, bool fWritable, bool fByPassHandlers)
5521	{
5522	PGM_LOCK_VOID(pVM);
5523	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
5524
5525	PPGMRAMRANGE pRam;
5526	PPGMPAGE pPage;
5527	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
5528	if (RT_SUCCESS(rc))
5529	{
5530	if (PGM_PAGE_IS_BALLOONED(pPage))
5531	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5532	else if (PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
5533	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5534	else if ( !PGM_PAGE_HAS_ANY_HANDLERS(pPage)
5535	\|\| (fByPassHandlers && !PGM_PAGE_IS_MMIO(pPage)) )
5536	rc = VINF_SUCCESS;
5537	else
5538	{
5539	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
5540	{
5541	Assert(!fByPassHandlers \|\| PGM_PAGE_IS_MMIO(pPage));
5542	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
5543	}
5544	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) && fWritable)
5545	{
5546	Assert(!fByPassHandlers);
5547	rc = VERR_PGM_PHYS_TLB_CATCH_WRITE;
5548	}
5549	}
5550	}
5551
5552	PGM_UNLOCK(pVM);
5553	return rc;
5554	}
5555
5556	#ifdef VBOX_WITH_NATIVE_NEM
5557
5558	/**
5559	* Interface used by NEM to check what to do on a memory access exit.
5560	*
5561	* @returns VBox status code.
5562	* @param pVM The cross context VM structure.
5563	* @param pVCpu The cross context per virtual CPU structure.
5564	* Optional.
5565	* @param GCPhys The guest physical address.
5566	* @param fMakeWritable Whether to try make the page writable or not. If it
5567	* cannot be made writable, NEM_PAGE_PROT_WRITE won't
5568	* be returned and the return code will be unaffected
5569	* @param pInfo Where to return the page information. This is
5570	* initialized even on failure.
5571	* @param pfnChecker Page in-sync checker callback. Optional.
5572	* @param pvUser User argument to pass to pfnChecker.
5573	*/
5574	VMM_INT_DECL(int) PGMPhysNemPageInfoChecker(PVMCC pVM, PVMCPUCC pVCpu, RTGCPHYS GCPhys, bool fMakeWritable,
5575	PPGMPHYSNEMPAGEINFO pInfo, PFNPGMPHYSNEMCHECKPAGE pfnChecker, void *pvUser)
5576	{
5577	PGM_LOCK_VOID(pVM);
5578
5579	PPGMPAGE pPage;
5580	int rc = pgmPhysGetPageEx(pVM, GCPhys, &pPage);
5581	if (RT_SUCCESS(rc))
5582	{
5583	/* Try make it writable if requested. */
5584	pInfo->u2OldNemState = PGM_PAGE_GET_NEM_STATE(pPage);
5585	if (fMakeWritable)
5586	switch (PGM_PAGE_GET_STATE(pPage))
5587	{
5588	case PGM_PAGE_STATE_SHARED:
5589	case PGM_PAGE_STATE_WRITE_MONITORED:
5590	case PGM_PAGE_STATE_ZERO:
5591	rc = pgmPhysPageMakeWritable(pVM, pPage, GCPhys);
5592	if (rc == VERR_PGM_PHYS_PAGE_RESERVED)
5593	rc = VINF_SUCCESS;
5594	break;
5595	}
5596
5597	/* Fill in the info. */
5598	pInfo->HCPhys = PGM_PAGE_GET_HCPHYS(pPage);
5599	pInfo->u2NemState = PGM_PAGE_GET_NEM_STATE(pPage);
5600	pInfo->fHasHandlers = PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage) ? 1 : 0;
5601	PGMPAGETYPE const enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5602	pInfo->enmType = enmType;
5603	pInfo->fNemProt = pgmPhysPageCalcNemProtection(pPage, enmType);
5604	switch (PGM_PAGE_GET_STATE(pPage))
5605	{
5606	case PGM_PAGE_STATE_ALLOCATED:
5607	pInfo->fZeroPage = 0;
5608	break;
5609
5610	case PGM_PAGE_STATE_ZERO:
5611	pInfo->fZeroPage = 1;
5612	break;
5613
5614	case PGM_PAGE_STATE_WRITE_MONITORED:
5615	pInfo->fZeroPage = 0;
5616	break;
5617
5618	case PGM_PAGE_STATE_SHARED:
5619	pInfo->fZeroPage = 0;
5620	break;
5621
5622	case PGM_PAGE_STATE_BALLOONED:
5623	pInfo->fZeroPage = 1;
5624	break;
5625
5626	default:
5627	pInfo->fZeroPage = 1;
5628	AssertFailedStmt(rc = VERR_PGM_PHYS_PAGE_GET_IPE);
5629	}
5630
5631	/* Call the checker and update NEM state. */
5632	if (pfnChecker)
5633	{
5634	rc = pfnChecker(pVM, pVCpu, GCPhys, pInfo, pvUser);
5635	PGM_PAGE_SET_NEM_STATE(pPage, pInfo->u2NemState);
5636	}
5637
5638	/* Done. */
5639	PGM_UNLOCK(pVM);
5640	}
5641	else
5642	{
5643	PGM_UNLOCK(pVM);
5644
5645	pInfo->HCPhys = NIL_RTHCPHYS;
5646	pInfo->fNemProt = NEM_PAGE_PROT_NONE;
5647	pInfo->u2NemState = 0;
5648	pInfo->fHasHandlers = 0;
5649	pInfo->fZeroPage = 0;
5650	pInfo->enmType = PGMPAGETYPE_INVALID;
5651	}
5652
5653	return rc;
5654	}
5655
5656
5657	/**
5658	* NEM helper that performs @a pfnCallback on pages with NEM state @a uMinState
5659	* or higher.
5660	*
5661	* @returns VBox status code from callback.
5662	* @param pVM The cross context VM structure.
5663	* @param pVCpu The cross context per CPU structure. This is
5664	* optional as its only for passing to callback.
5665	* @param uMinState The minimum NEM state value to call on.
5666	* @param pfnCallback The callback function.
5667	* @param pvUser User argument for the callback.
5668	*/
5669	VMM_INT_DECL(int) PGMPhysNemEnumPagesByState(PVMCC pVM, PVMCPUCC pVCpu, uint8_t uMinState,
5670	PFNPGMPHYSNEMENUMCALLBACK pfnCallback, void *pvUser)
5671	{
5672	/*
5673	* Just brute force this problem.
5674	*/
5675	PGM_LOCK_VOID(pVM);
5676	int rc = VINF_SUCCESS;
5677	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
5678	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries && RT_SUCCESS(rc); idxLookup++)
5679	{
5680	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
5681	AssertContinue(idRamRange < RT_ELEMENTS(pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges));
5682	PPGMRAMRANGE const pRam = pVM->CTX_EXPR(pgm, pgmr0, pgm).s.apRamRanges[idRamRange];
5683	AssertContinue(pRam);
5684	Assert(pRam->GCPhys == PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]));
5685
5686	#ifdef IN_RING0
5687	uint32_t const cPages = RT_MIN(pRam->cb >> X86_PAGE_SHIFT, pVM->pgmr0.s.acRamRangePages[idRamRange]);
5688	#else
5689	uint32_t const cPages = pRam->cb >> X86_PAGE_SHIFT;
5690	#endif
5691	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5692	{
5693	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(&pRam->aPages[iPage]);
5694	if (u2State < uMinState)
5695	{ /* likely */ }
5696	else
5697	{
5698	rc = pfnCallback(pVM, pVCpu, pRam->GCPhys + ((RTGCPHYS)iPage << X86_PAGE_SHIFT), &u2State, pvUser);
5699	if (RT_SUCCESS(rc))
5700	PGM_PAGE_SET_NEM_STATE(&pRam->aPages[iPage], u2State);
5701	else
5702	break;
5703	}
5704	}
5705	}
5706	PGM_UNLOCK(pVM);
5707
5708	return rc;
5709	}
5710
5711
5712	/**
5713	* Helper for setting the NEM state for a range of pages.
5714	*
5715	* @param paPages Array of pages to modify.
5716	* @param cPages How many pages to modify.
5717	* @param u2State The new state value.
5718	*/
5719	DECLHIDDEN(void) pgmPhysSetNemStateForPages(PPGMPAGE paPages, RTGCPHYS cPages, uint8_t u2State)
5720	{
5721	PPGMPAGE pPage = paPages;
5722	while (cPages-- > 0)
5723	{
5724	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
5725	pPage++;
5726	}
5727	}
5728
5729	#endif /* VBOX_WITH_NATIVE_NEM */
5730

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/PGMAllPhys.cpp

Download in other formats: