PGMPhys.cpp@ 104840

Last change on this file since 104840 was 104840, checked in by vboxsync, 12 months ago
VMM/PGM: Refactored RAM ranges, MMIO2 ranges and ROM ranges and added MMIO ranges (to PGM) so we can safely access RAM ranges at runtime w/o fear of them ever being freed up. It is now only possible to create these during VM creation and loading, and they will live till VM destruction (except for MMIO2 which could be destroyed during loading (PCNet fun)). The lookup handling is by table instead of pointer tree. No more ring-0 pointers in shared data. bugref:10687 bugref:10093
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 245.3 KB

Line
1	/* $Id: PGMPhys.cpp 104840 2024-06-05 00:59:51Z vboxsync $ */
2	/** @file
3	* PGM - Page Manager and Monitor, Physical Memory Addressing.
4	*/
5
6	/*
7	* Copyright (C) 2006-2023 Oracle and/or its affiliates.
8	*
9	* This file is part of VirtualBox base platform packages, as
10	* available from https://www.215389.xyz.
11	*
12	* This program is free software; you can redistribute it and/or
13	* modify it under the terms of the GNU General Public License
14	* as published by the Free Software Foundation, in version 3 of the
15	* License.
16	*
17	* This program is distributed in the hope that it will be useful, but
18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	* General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with this program; if not, see <https://www.gnu.org/licenses>.
24	*
25	* SPDX-License-Identifier: GPL-3.0-only
26	*/
27
28
29	/*********************************************************************************************************************************
30	* Header Files *
31	*********************************************************************************************************************************/
32	#define LOG_GROUP LOG_GROUP_PGM_PHYS
33	#define VBOX_WITHOUT_PAGING_BIT_FIELDS /* 64-bit bitfields are just asking for trouble. See @bugref{9841} and others. */
34	#include <VBox/vmm/pgm.h>
35	#include <VBox/vmm/iem.h>
36	#include <VBox/vmm/iom.h>
37	#include <VBox/vmm/mm.h>
38	#include <VBox/vmm/nem.h>
39	#include <VBox/vmm/stam.h>
40	#include <VBox/vmm/pdmdev.h>
41	#include "PGMInternal.h"
42	#include <VBox/vmm/vmcc.h>
43
44	#include "PGMInline.h"
45
46	#include <VBox/sup.h>
47	#include <VBox/param.h>
48	#include <VBox/err.h>
49	#include <VBox/log.h>
50	#include <iprt/assert.h>
51	#include <iprt/alloc.h>
52	#include <iprt/asm.h>
53	#ifdef VBOX_STRICT
54	# include <iprt/crc.h>
55	#endif
56	#include <iprt/thread.h>
57	#include <iprt/string.h>
58	#include <iprt/system.h>
59
60
61	/*********************************************************************************************************************************
62	* Defined Constants And Macros *
63	*********************************************************************************************************************************/
64	/** The number of pages to free in one batch. */
65	#define PGMPHYS_FREE_PAGE_BATCH_SIZE 128
66
67
68
69	/*********************************************************************************************************************************
70	* Reading and Writing Guest Pysical Memory *
71	*********************************************************************************************************************************/
72
73	/*
74	* PGMR3PhysReadU8-64
75	* PGMR3PhysWriteU8-64
76	*/
77	#define PGMPHYSFN_READNAME PGMR3PhysReadU8
78	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU8
79	#define PGMPHYS_DATASIZE 1
80	#define PGMPHYS_DATATYPE uint8_t
81	#include "PGMPhysRWTmpl.h"
82
83	#define PGMPHYSFN_READNAME PGMR3PhysReadU16
84	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU16
85	#define PGMPHYS_DATASIZE 2
86	#define PGMPHYS_DATATYPE uint16_t
87	#include "PGMPhysRWTmpl.h"
88
89	#define PGMPHYSFN_READNAME PGMR3PhysReadU32
90	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU32
91	#define PGMPHYS_DATASIZE 4
92	#define PGMPHYS_DATATYPE uint32_t
93	#include "PGMPhysRWTmpl.h"
94
95	#define PGMPHYSFN_READNAME PGMR3PhysReadU64
96	#define PGMPHYSFN_WRITENAME PGMR3PhysWriteU64
97	#define PGMPHYS_DATASIZE 8
98	#define PGMPHYS_DATATYPE uint64_t
99	#include "PGMPhysRWTmpl.h"
100
101
102	/**
103	* EMT worker for PGMR3PhysReadExternal.
104	*/
105	static DECLCALLBACK(int) pgmR3PhysReadExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, void *pvBuf, size_t cbRead,
106	PGMACCESSORIGIN enmOrigin)
107	{
108	VBOXSTRICTRC rcStrict = PGMPhysRead(pVM, *pGCPhys, pvBuf, cbRead, enmOrigin);
109	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
110	return VINF_SUCCESS;
111	}
112
113
114	/**
115	* Read from physical memory, external users.
116	*
117	* @returns VBox status code.
118	* @retval VINF_SUCCESS.
119	*
120	* @param pVM The cross context VM structure.
121	* @param GCPhys Physical address to read from.
122	* @param pvBuf Where to read into.
123	* @param cbRead How many bytes to read.
124	* @param enmOrigin Who is calling.
125	*
126	* @thread Any but EMTs.
127	*/
128	VMMR3DECL(int) PGMR3PhysReadExternal(PVM pVM, RTGCPHYS GCPhys, void *pvBuf, size_t cbRead, PGMACCESSORIGIN enmOrigin)
129	{
130	VM_ASSERT_OTHER_THREAD(pVM);
131
132	AssertMsgReturn(cbRead > 0, ("don't even think about reading zero bytes!\n"), VINF_SUCCESS);
133	LogFlow(("PGMR3PhysReadExternal: %RGp %d\n", GCPhys, cbRead));
134
135	PGM_LOCK_VOID(pVM);
136
137	/*
138	* Copy loop on ram ranges.
139	*/
140	for (;;)
141	{
142	PPGMRAMRANGE pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
143
144	/* Inside range or not? */
145	if (pRam && GCPhys >= pRam->GCPhys)
146	{
147	/*
148	* Must work our way thru this page by page.
149	*/
150	RTGCPHYS off = GCPhys - pRam->GCPhys;
151	while (off < pRam->cb)
152	{
153	unsigned iPage = off >> GUEST_PAGE_SHIFT;
154	PPGMPAGE pPage = &pRam->aPages[iPage];
155
156	/*
157	* If the page has an ALL access handler, we'll have to
158	* delegate the job to EMT.
159	*/
160	if ( PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)
161	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
162	{
163	PGM_UNLOCK(pVM);
164
165	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysReadExternalEMT, 5,
166	pVM, &GCPhys, pvBuf, cbRead, enmOrigin);
167	}
168	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
169
170	/*
171	* Simple stuff, go ahead.
172	*/
173	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
174	if (cb > cbRead)
175	cb = cbRead;
176	PGMPAGEMAPLOCK PgMpLck;
177	const void *pvSrc;
178	int rc = pgmPhysGCPhys2CCPtrInternalReadOnly(pVM, pPage, pRam->GCPhys + off, &pvSrc, &PgMpLck);
179	if (RT_SUCCESS(rc))
180	{
181	memcpy(pvBuf, pvSrc, cb);
182	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
183	}
184	else
185	{
186	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternalReadOnly failed on %RGp / %R[pgmpage] -> %Rrc\n",
187	pRam->GCPhys + off, pPage, rc));
188	memset(pvBuf, 0xff, cb);
189	}
190
191	/* next page */
192	if (cb >= cbRead)
193	{
194	PGM_UNLOCK(pVM);
195	return VINF_SUCCESS;
196	}
197	cbRead -= cb;
198	off += cb;
199	GCPhys += cb;
200	pvBuf = (char *)pvBuf + cb;
201	} /* walk pages in ram range. */
202	}
203	else
204	{
205	LogFlow(("PGMPhysRead: Unassigned %RGp size=%u\n", GCPhys, cbRead));
206
207	/*
208	* Unassigned address space.
209	*/
210	size_t cb = pRam ? pRam->GCPhys - GCPhys : ~(size_t)0;
211	if (cb >= cbRead)
212	{
213	memset(pvBuf, 0xff, cbRead);
214	break;
215	}
216	memset(pvBuf, 0xff, cb);
217
218	cbRead -= cb;
219	pvBuf = (char *)pvBuf + cb;
220	GCPhys += cb;
221	}
222	} /* Ram range walk */
223
224	PGM_UNLOCK(pVM);
225
226	return VINF_SUCCESS;
227	}
228
229
230	/**
231	* EMT worker for PGMR3PhysWriteExternal.
232	*/
233	static DECLCALLBACK(int) pgmR3PhysWriteExternalEMT(PVM pVM, PRTGCPHYS pGCPhys, const void *pvBuf, size_t cbWrite,
234	PGMACCESSORIGIN enmOrigin)
235	{
236	/** @todo VERR_EM_NO_MEMORY */
237	VBOXSTRICTRC rcStrict = PGMPhysWrite(pVM, *pGCPhys, pvBuf, cbWrite, enmOrigin);
238	AssertMsg(rcStrict == VINF_SUCCESS, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict))); NOREF(rcStrict);
239	return VINF_SUCCESS;
240	}
241
242
243	/**
244	* Write to physical memory, external users.
245	*
246	* @returns VBox status code.
247	* @retval VINF_SUCCESS.
248	* @retval VERR_EM_NO_MEMORY.
249	*
250	* @param pVM The cross context VM structure.
251	* @param GCPhys Physical address to write to.
252	* @param pvBuf What to write.
253	* @param cbWrite How many bytes to write.
254	* @param enmOrigin Who is calling.
255	*
256	* @thread Any but EMTs.
257	*/
258	VMMDECL(int) PGMR3PhysWriteExternal(PVM pVM, RTGCPHYS GCPhys, const void *pvBuf, size_t cbWrite, PGMACCESSORIGIN enmOrigin)
259	{
260	VM_ASSERT_OTHER_THREAD(pVM);
261
262	AssertMsg(!pVM->pgm.s.fNoMorePhysWrites,
263	("Calling PGMR3PhysWriteExternal after pgmR3Save()! GCPhys=%RGp cbWrite=%#x enmOrigin=%d\n",
264	GCPhys, cbWrite, enmOrigin));
265	AssertMsgReturn(cbWrite > 0, ("don't even think about writing zero bytes!\n"), VINF_SUCCESS);
266	LogFlow(("PGMR3PhysWriteExternal: %RGp %d\n", GCPhys, cbWrite));
267
268	PGM_LOCK_VOID(pVM);
269
270	/*
271	* Copy loop on ram ranges, stop when we hit something difficult.
272	*/
273	for (;;)
274	{
275	PPGMRAMRANGE const pRam = pgmPhysGetRangeAtOrAbove(pVM, GCPhys);
276
277	/* Inside range or not? */
278	if (pRam && GCPhys >= pRam->GCPhys)
279	{
280	/*
281	* Must work our way thru this page by page.
282	*/
283	RTGCPTR off = GCPhys - pRam->GCPhys;
284	while (off < pRam->cb)
285	{
286	RTGCPTR iPage = off >> GUEST_PAGE_SHIFT;
287	PPGMPAGE pPage = &pRam->aPages[iPage];
288
289	/*
290	* Is the page problematic, we have to do the work on the EMT.
291	*
292	* Allocating writable pages and access handlers are
293	* problematic, write monitored pages are simple and can be
294	* dealt with here.
295	*/
296	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
297	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
298	\|\| PGM_PAGE_IS_SPECIAL_ALIAS_MMIO(pPage))
299	{
300	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
301	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
302	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
303	else
304	{
305	PGM_UNLOCK(pVM);
306
307	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysWriteExternalEMT, 5,
308	pVM, &GCPhys, pvBuf, cbWrite, enmOrigin);
309	}
310	}
311	Assert(!PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage));
312
313	/*
314	* Simple stuff, go ahead.
315	*/
316	size_t cb = GUEST_PAGE_SIZE - (off & GUEST_PAGE_OFFSET_MASK);
317	if (cb > cbWrite)
318	cb = cbWrite;
319	PGMPAGEMAPLOCK PgMpLck;
320	void *pvDst;
321	int rc = pgmPhysGCPhys2CCPtrInternal(pVM, pPage, pRam->GCPhys + off, &pvDst, &PgMpLck);
322	if (RT_SUCCESS(rc))
323	{
324	memcpy(pvDst, pvBuf, cb);
325	pgmPhysReleaseInternalPageMappingLock(pVM, &PgMpLck);
326	}
327	else
328	AssertLogRelMsgFailed(("pgmPhysGCPhys2CCPtrInternal failed on %RGp / %R[pgmpage] -> %Rrc\n",
329	pRam->GCPhys + off, pPage, rc));
330
331	/* next page */
332	if (cb >= cbWrite)
333	{
334	PGM_UNLOCK(pVM);
335	return VINF_SUCCESS;
336	}
337
338	cbWrite -= cb;
339	off += cb;
340	GCPhys += cb;
341	pvBuf = (const char *)pvBuf + cb;
342	} /* walk pages in ram range */
343	}
344	else
345	{
346	/*
347	* Unassigned address space, skip it.
348	*/
349	if (!pRam)
350	break;
351	size_t cb = pRam->GCPhys - GCPhys;
352	if (cb >= cbWrite)
353	break;
354	cbWrite -= cb;
355	pvBuf = (const char *)pvBuf + cb;
356	GCPhys += cb;
357	}
358	} /* Ram range walk */
359
360	PGM_UNLOCK(pVM);
361	return VINF_SUCCESS;
362	}
363
364
365	/*********************************************************************************************************************************
366	* Mapping Guest Physical Memory *
367	*********************************************************************************************************************************/
368
369	/**
370	* VMR3ReqCall worker for PGMR3PhysGCPhys2CCPtrExternal to make pages writable.
371	*
372	* @returns see PGMR3PhysGCPhys2CCPtrExternal
373	* @param pVM The cross context VM structure.
374	* @param pGCPhys Pointer to the guest physical address.
375	* @param ppv Where to store the mapping address.
376	* @param pLock Where to store the lock.
377	*/
378	static DECLCALLBACK(int) pgmR3PhysGCPhys2CCPtrDelegated(PVM pVM, PRTGCPHYS pGCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
379	{
380	/*
381	* Just hand it to PGMPhysGCPhys2CCPtr and check that it's not a page with
382	* an access handler after it succeeds.
383	*/
384	int rc = PGM_LOCK(pVM);
385	AssertRCReturn(rc, rc);
386
387	rc = PGMPhysGCPhys2CCPtr(pVM, *pGCPhys, ppv, pLock);
388	if (RT_SUCCESS(rc))
389	{
390	PPGMPAGEMAPTLBE pTlbe;
391	int rc2 = pgmPhysPageQueryTlbe(pVM, *pGCPhys, &pTlbe);
392	AssertFatalRC(rc2);
393	PPGMPAGE pPage = pTlbe->pPage;
394	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
395	{
396	PGMPhysReleasePageMappingLock(pVM, pLock);
397	rc = VERR_PGM_PHYS_PAGE_RESERVED;
398	}
399	else if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
400	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
401	\|\| pgmPoolIsDirtyPage(pVM, *pGCPhys)
402	#endif
403	)
404	{
405	/* We must flush any corresponding pgm pool page here, otherwise we'll
406	* not be informed about writes and keep bogus gst->shw mappings around.
407	*/
408	pgmPoolFlushPageByGCPhys(pVM, *pGCPhys);
409	Assert(!PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage));
410	/** @todo r=bird: return VERR_PGM_PHYS_PAGE_RESERVED here if it still has
411	* active handlers, see the PGMR3PhysGCPhys2CCPtrExternal docs. */
412	}
413	}
414
415	PGM_UNLOCK(pVM);
416	return rc;
417	}
418
419
420	/**
421	* Requests the mapping of a guest page into ring-3, external threads.
422	*
423	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
424	* release it.
425	*
426	* This API will assume your intention is to write to the page, and will
427	* therefore replace shared and zero pages. If you do not intend to modify the
428	* page, use the PGMR3PhysGCPhys2CCPtrReadOnlyExternal() API.
429	*
430	* @returns VBox status code.
431	* @retval VINF_SUCCESS on success.
432	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
433	* backing or if the page has any active access handlers. The caller
434	* must fall back on using PGMR3PhysWriteExternal.
435	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
436	*
437	* @param pVM The cross context VM structure.
438	* @param GCPhys The guest physical address of the page that should be mapped.
439	* @param ppv Where to store the address corresponding to GCPhys.
440	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
441	*
442	* @remark Avoid calling this API from within critical sections (other than the
443	* PGM one) because of the deadlock risk when we have to delegating the
444	* task to an EMT.
445	* @thread Any.
446	*/
447	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrExternal(PVM pVM, RTGCPHYS GCPhys, void **ppv, PPGMPAGEMAPLOCK pLock)
448	{
449	AssertPtr(ppv);
450	AssertPtr(pLock);
451
452	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
453
454	int rc = PGM_LOCK(pVM);
455	AssertRCReturn(rc, rc);
456
457	/*
458	* Query the Physical TLB entry for the page (may fail).
459	*/
460	PPGMPAGEMAPTLBE pTlbe;
461	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
462	if (RT_SUCCESS(rc))
463	{
464	PPGMPAGE pPage = pTlbe->pPage;
465	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
466	rc = VERR_PGM_PHYS_PAGE_RESERVED;
467	else
468	{
469	/*
470	* If the page is shared, the zero page, or being write monitored
471	* it must be converted to an page that's writable if possible.
472	* We can only deal with write monitored pages here, the rest have
473	* to be on an EMT.
474	*/
475	if ( PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
476	\|\| PGM_PAGE_GET_STATE(pPage) != PGM_PAGE_STATE_ALLOCATED
477	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
478	\|\| pgmPoolIsDirtyPage(pVM, GCPhys)
479	#endif
480	)
481	{
482	if ( PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED
483	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage)
484	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
485	&& !pgmPoolIsDirtyPage(pVM, GCPhys) /** @todo we're very likely doing this twice. */
486	#endif
487	)
488	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, GCPhys);
489	else
490	{
491	PGM_UNLOCK(pVM);
492
493	return VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
494	pVM, &GCPhys, ppv, pLock);
495	}
496	}
497
498	/*
499	* Now, just perform the locking and calculate the return address.
500	*/
501	PPGMPAGEMAP pMap = pTlbe->pMap;
502	if (pMap)
503	pMap->cRefs++;
504
505	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
506	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
507	{
508	if (cLocks == 0)
509	pVM->pgm.s.cWriteLockedPages++;
510	PGM_PAGE_INC_WRITE_LOCKS(pPage);
511	}
512	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
513	{
514	PGM_PAGE_INC_WRITE_LOCKS(pPage);
515	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", GCPhys, pPage));
516	if (pMap)
517	pMap->cRefs++; /* Extra ref to prevent it from going away. */
518	}
519
520	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
521	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
522	pLock->pvMap = pMap;
523	}
524	}
525
526	PGM_UNLOCK(pVM);
527	return rc;
528	}
529
530
531	/**
532	* Requests the mapping of a guest page into ring-3, external threads.
533	*
534	* When you're done with the page, call PGMPhysReleasePageMappingLock() ASAP to
535	* release it.
536	*
537	* @returns VBox status code.
538	* @retval VINF_SUCCESS on success.
539	* @retval VERR_PGM_PHYS_PAGE_RESERVED it it's a valid page but has no physical
540	* backing or if the page as an active ALL access handler. The caller
541	* must fall back on using PGMPhysRead.
542	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if it's not a valid physical address.
543	*
544	* @param pVM The cross context VM structure.
545	* @param GCPhys The guest physical address of the page that should be mapped.
546	* @param ppv Where to store the address corresponding to GCPhys.
547	* @param pLock Where to store the lock information that PGMPhysReleasePageMappingLock needs.
548	*
549	* @remark Avoid calling this API from within critical sections (other than
550	* the PGM one) because of the deadlock risk.
551	* @thread Any.
552	*/
553	VMMR3DECL(int) PGMR3PhysGCPhys2CCPtrReadOnlyExternal(PVM pVM, RTGCPHYS GCPhys, void const **ppv, PPGMPAGEMAPLOCK pLock)
554	{
555	int rc = PGM_LOCK(pVM);
556	AssertRCReturn(rc, rc);
557
558	/*
559	* Query the Physical TLB entry for the page (may fail).
560	*/
561	PPGMPAGEMAPTLBE pTlbe;
562	rc = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
563	if (RT_SUCCESS(rc))
564	{
565	PPGMPAGE pPage = pTlbe->pPage;
566	#if 1
567	/* MMIO pages doesn't have any readable backing. */
568	if (PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage))
569	rc = VERR_PGM_PHYS_PAGE_RESERVED;
570	#else
571	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
572	rc = VERR_PGM_PHYS_PAGE_RESERVED;
573	#endif
574	else
575	{
576	/*
577	* Now, just perform the locking and calculate the return address.
578	*/
579	PPGMPAGEMAP pMap = pTlbe->pMap;
580	if (pMap)
581	pMap->cRefs++;
582
583	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
584	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
585	{
586	if (cLocks == 0)
587	pVM->pgm.s.cReadLockedPages++;
588	PGM_PAGE_INC_READ_LOCKS(pPage);
589	}
590	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
591	{
592	PGM_PAGE_INC_READ_LOCKS(pPage);
593	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", GCPhys, pPage));
594	if (pMap)
595	pMap->cRefs++; /* Extra ref to prevent it from going away. */
596	}
597
598	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
599	pLock->uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
600	pLock->pvMap = pMap;
601	}
602	}
603
604	PGM_UNLOCK(pVM);
605	return rc;
606	}
607
608
609	/**
610	* Requests the mapping of multiple guest page into ring-3, external threads.
611	*
612	* When you're done with the pages, call PGMPhysBulkReleasePageMappingLock()
613	* ASAP to release them.
614	*
615	* This API will assume your intention is to write to the pages, and will
616	* therefore replace shared and zero pages. If you do not intend to modify the
617	* pages, use the PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal() API.
618	*
619	* @returns VBox status code.
620	* @retval VINF_SUCCESS on success.
621	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
622	* backing or if any of the pages the page has any active access
623	* handlers. The caller must fall back on using PGMR3PhysWriteExternal.
624	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
625	* an invalid physical address.
626	*
627	* @param pVM The cross context VM structure.
628	* @param cPages Number of pages to lock.
629	* @param paGCPhysPages The guest physical address of the pages that
630	* should be mapped (@a cPages entries).
631	* @param papvPages Where to store the ring-3 mapping addresses
632	* corresponding to @a paGCPhysPages.
633	* @param paLocks Where to store the locking information that
634	* pfnPhysBulkReleasePageMappingLock needs (@a cPages
635	* in length).
636	*
637	* @remark Avoid calling this API from within critical sections (other than the
638	* PGM one) because of the deadlock risk when we have to delegating the
639	* task to an EMT.
640	* @thread Any.
641	*/
642	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
643	void **papvPages, PPGMPAGEMAPLOCK paLocks)
644	{
645	Assert(cPages > 0);
646	AssertPtr(papvPages);
647	AssertPtr(paLocks);
648
649	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
650
651	int rc = PGM_LOCK(pVM);
652	AssertRCReturn(rc, rc);
653
654	/*
655	* Lock the pages one by one.
656	* The loop body is similar to PGMR3PhysGCPhys2CCPtrExternal.
657	*/
658	int32_t cNextYield = 128;
659	uint32_t iPage;
660	for (iPage = 0; iPage < cPages; iPage++)
661	{
662	if (--cNextYield > 0)
663	{ /* likely */ }
664	else
665	{
666	PGM_UNLOCK(pVM);
667	ASMNopPause();
668	PGM_LOCK_VOID(pVM);
669	cNextYield = 128;
670	}
671
672	/*
673	* Query the Physical TLB entry for the page (may fail).
674	*/
675	PPGMPAGEMAPTLBE pTlbe;
676	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
677	if (RT_SUCCESS(rc))
678	{ }
679	else
680	break;
681	PPGMPAGE pPage = pTlbe->pPage;
682
683	/*
684	* No MMIO or active access handlers.
685	*/
686	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
687	&& !PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
688	{ }
689	else
690	{
691	rc = VERR_PGM_PHYS_PAGE_RESERVED;
692	break;
693	}
694
695	/*
696	* The page must be in the allocated state and not be a dirty pool page.
697	* We can handle converting a write monitored page to an allocated one, but
698	* anything more complicated must be delegated to an EMT.
699	*/
700	bool fDelegateToEmt = false;
701	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_ALLOCATED)
702	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
703	fDelegateToEmt = pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]);
704	#else
705	fDelegateToEmt = false;
706	#endif
707	else if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
708	{
709	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
710	if (!pgmPoolIsDirtyPage(pVM, paGCPhysPages[iPage]))
711	pgmPhysPageMakeWriteMonitoredWritable(pVM, pPage, paGCPhysPages[iPage]);
712	else
713	fDelegateToEmt = true;
714	#endif
715	}
716	else
717	fDelegateToEmt = true;
718	if (!fDelegateToEmt)
719	{ }
720	else
721	{
722	/* We could do this delegation in bulk, but considered too much work vs gain. */
723	PGM_UNLOCK(pVM);
724	rc = VMR3ReqPriorityCallWait(pVM, VMCPUID_ANY, (PFNRT)pgmR3PhysGCPhys2CCPtrDelegated, 4,
725	pVM, &paGCPhysPages[iPage], &papvPages[iPage], &paLocks[iPage]);
726	PGM_LOCK_VOID(pVM);
727	if (RT_FAILURE(rc))
728	break;
729	cNextYield = 128;
730	}
731
732	/*
733	* Now, just perform the locking and address calculation.
734	*/
735	PPGMPAGEMAP pMap = pTlbe->pMap;
736	if (pMap)
737	pMap->cRefs++;
738
739	unsigned cLocks = PGM_PAGE_GET_WRITE_LOCKS(pPage);
740	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
741	{
742	if (cLocks == 0)
743	pVM->pgm.s.cWriteLockedPages++;
744	PGM_PAGE_INC_WRITE_LOCKS(pPage);
745	}
746	else if (cLocks != PGM_PAGE_GET_WRITE_LOCKS(pPage))
747	{
748	PGM_PAGE_INC_WRITE_LOCKS(pPage);
749	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent write locked state!\n", paGCPhysPages[iPage], pPage));
750	if (pMap)
751	pMap->cRefs++; /* Extra ref to prevent it from going away. */
752	}
753
754	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
755	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_WRITE;
756	paLocks[iPage].pvMap = pMap;
757	}
758
759	PGM_UNLOCK(pVM);
760
761	/*
762	* On failure we must unlock any pages we managed to get already.
763	*/
764	if (RT_FAILURE(rc) && iPage > 0)
765	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
766
767	return rc;
768	}
769
770
771	/**
772	* Requests the mapping of multiple guest page into ring-3, for reading only,
773	* external threads.
774	*
775	* When you're done with the pages, call PGMPhysReleasePageMappingLock() ASAP
776	* to release them.
777	*
778	* @returns VBox status code.
779	* @retval VINF_SUCCESS on success.
780	* @retval VERR_PGM_PHYS_PAGE_RESERVED if any of the pages has no physical
781	* backing or if any of the pages the page has an active ALL access
782	* handler. The caller must fall back on using PGMR3PhysWriteExternal.
783	* @retval VERR_PGM_INVALID_GC_PHYSICAL_ADDRESS if @a paGCPhysPages contains
784	* an invalid physical address.
785	*
786	* @param pVM The cross context VM structure.
787	* @param cPages Number of pages to lock.
788	* @param paGCPhysPages The guest physical address of the pages that
789	* should be mapped (@a cPages entries).
790	* @param papvPages Where to store the ring-3 mapping addresses
791	* corresponding to @a paGCPhysPages.
792	* @param paLocks Where to store the lock information that
793	* pfnPhysReleasePageMappingLock needs (@a cPages
794	* in length).
795	*
796	* @remark Avoid calling this API from within critical sections (other than
797	* the PGM one) because of the deadlock risk.
798	* @thread Any.
799	*/
800	VMMR3DECL(int) PGMR3PhysBulkGCPhys2CCPtrReadOnlyExternal(PVM pVM, uint32_t cPages, PCRTGCPHYS paGCPhysPages,
801	void const **papvPages, PPGMPAGEMAPLOCK paLocks)
802	{
803	Assert(cPages > 0);
804	AssertPtr(papvPages);
805	AssertPtr(paLocks);
806
807	Assert(VM_IS_EMT(pVM) \|\| !PGMIsLockOwner(pVM));
808
809	int rc = PGM_LOCK(pVM);
810	AssertRCReturn(rc, rc);
811
812	/*
813	* Lock the pages one by one.
814	* The loop body is similar to PGMR3PhysGCPhys2CCPtrReadOnlyExternal.
815	*/
816	int32_t cNextYield = 256;
817	uint32_t iPage;
818	for (iPage = 0; iPage < cPages; iPage++)
819	{
820	if (--cNextYield > 0)
821	{ /* likely */ }
822	else
823	{
824	PGM_UNLOCK(pVM);
825	ASMNopPause();
826	PGM_LOCK_VOID(pVM);
827	cNextYield = 256;
828	}
829
830	/*
831	* Query the Physical TLB entry for the page (may fail).
832	*/
833	PPGMPAGEMAPTLBE pTlbe;
834	rc = pgmPhysPageQueryTlbe(pVM, paGCPhysPages[iPage], &pTlbe);
835	if (RT_SUCCESS(rc))
836	{ }
837	else
838	break;
839	PPGMPAGE pPage = pTlbe->pPage;
840
841	/*
842	* No MMIO or active all access handlers, everything else can be accessed.
843	*/
844	if ( !PGM_PAGE_IS_MMIO_OR_SPECIAL_ALIAS(pPage)
845	&& !PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage))
846	{ }
847	else
848	{
849	rc = VERR_PGM_PHYS_PAGE_RESERVED;
850	break;
851	}
852
853	/*
854	* Now, just perform the locking and address calculation.
855	*/
856	PPGMPAGEMAP pMap = pTlbe->pMap;
857	if (pMap)
858	pMap->cRefs++;
859
860	unsigned cLocks = PGM_PAGE_GET_READ_LOCKS(pPage);
861	if (RT_LIKELY(cLocks < PGM_PAGE_MAX_LOCKS - 1))
862	{
863	if (cLocks == 0)
864	pVM->pgm.s.cReadLockedPages++;
865	PGM_PAGE_INC_READ_LOCKS(pPage);
866	}
867	else if (cLocks != PGM_PAGE_GET_READ_LOCKS(pPage))
868	{
869	PGM_PAGE_INC_READ_LOCKS(pPage);
870	AssertMsgFailed(("%RGp / %R[pgmpage] is entering permanent readonly locked state!\n", paGCPhysPages[iPage], pPage));
871	if (pMap)
872	pMap->cRefs++; /* Extra ref to prevent it from going away. */
873	}
874
875	papvPages[iPage] = (void *)((uintptr_t)pTlbe->pv \| (uintptr_t)(paGCPhysPages[iPage] & GUEST_PAGE_OFFSET_MASK));
876	paLocks[iPage].uPageAndType = (uintptr_t)pPage \| PGMPAGEMAPLOCK_TYPE_READ;
877	paLocks[iPage].pvMap = pMap;
878	}
879
880	PGM_UNLOCK(pVM);
881
882	/*
883	* On failure we must unlock any pages we managed to get already.
884	*/
885	if (RT_FAILURE(rc) && iPage > 0)
886	PGMPhysBulkReleasePageMappingLocks(pVM, iPage, paLocks);
887
888	return rc;
889	}
890
891
892	/**
893	* Converts a GC physical address to a HC ring-3 pointer, with some
894	* additional checks.
895	*
896	* @returns VBox status code.
897	* @retval VINF_SUCCESS on success.
898	* @retval VINF_PGM_PHYS_TLB_CATCH_WRITE and *ppv set if the page has a write
899	* access handler of some kind.
900	* @retval VERR_PGM_PHYS_TLB_CATCH_ALL if the page has a handler catching all
901	* accesses or is odd in any way.
902	* @retval VERR_PGM_PHYS_TLB_UNASSIGNED if the page doesn't exist.
903	*
904	* @param pVM The cross context VM structure.
905	* @param GCPhys The GC physical address to convert. Since this is only
906	* used for filling the REM TLB, the A20 mask must be
907	* applied before calling this API.
908	* @param fWritable Whether write access is required.
909	* @param ppv Where to store the pointer corresponding to GCPhys on
910	* success.
911	*/
912	VMMR3DECL(int) PGMR3PhysTlbGCPhys2Ptr(PVM pVM, RTGCPHYS GCPhys, bool fWritable, void **ppv)
913	{
914	PGM_LOCK_VOID(pVM);
915	PGM_A20_ASSERT_MASKED(VMMGetCpu(pVM), GCPhys);
916
917	PPGMRAMRANGE pRam;
918	PPGMPAGE pPage;
919	int rc = pgmPhysGetPageAndRangeEx(pVM, GCPhys, &pPage, &pRam);
920	if (RT_SUCCESS(rc))
921	{
922	if (PGM_PAGE_IS_BALLOONED(pPage))
923	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
924	else if (!PGM_PAGE_HAS_ANY_HANDLERS(pPage))
925	rc = VINF_SUCCESS;
926	else
927	{
928	if (PGM_PAGE_HAS_ACTIVE_ALL_HANDLERS(pPage)) /* catches MMIO */
929	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
930	else if (PGM_PAGE_HAS_ACTIVE_HANDLERS(pPage))
931	{
932	/** @todo Handle TLB loads of virtual handlers so ./test.sh can be made to work
933	* in -norawr0 mode. */
934	if (fWritable)
935	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
936	}
937	else
938	{
939	/* Temporarily disabled physical handler(s), since the recompiler
940	doesn't get notified when it's reset we'll have to pretend it's
941	operating normally. */
942	if (pgmHandlerPhysicalIsAll(pVM, GCPhys))
943	rc = VERR_PGM_PHYS_TLB_CATCH_ALL;
944	else
945	rc = VINF_PGM_PHYS_TLB_CATCH_WRITE;
946	}
947	}
948	if (RT_SUCCESS(rc))
949	{
950	int rc2;
951
952	/* Make sure what we return is writable. */
953	if (fWritable)
954	switch (PGM_PAGE_GET_STATE(pPage))
955	{
956	case PGM_PAGE_STATE_ALLOCATED:
957	break;
958	case PGM_PAGE_STATE_BALLOONED:
959	AssertFailed();
960	break;
961	case PGM_PAGE_STATE_ZERO:
962	case PGM_PAGE_STATE_SHARED:
963	if (rc == VINF_PGM_PHYS_TLB_CATCH_WRITE)
964	break;
965	RT_FALL_THRU();
966	case PGM_PAGE_STATE_WRITE_MONITORED:
967	rc2 = pgmPhysPageMakeWritable(pVM, pPage, GCPhys & ~(RTGCPHYS)GUEST_PAGE_OFFSET_MASK);
968	AssertLogRelRCReturn(rc2, rc2);
969	break;
970	}
971
972	/* Get a ring-3 mapping of the address. */
973	PPGMPAGER3MAPTLBE pTlbe;
974	rc2 = pgmPhysPageQueryTlbe(pVM, GCPhys, &pTlbe);
975	AssertLogRelRCReturn(rc2, rc2);
976	ppv = (void )((uintptr_t)pTlbe->pv \| (uintptr_t)(GCPhys & GUEST_PAGE_OFFSET_MASK));
977	/** @todo mapping/locking hell; this isn't horribly efficient since
978	* pgmPhysPageLoadIntoTlb will repeat the lookup we've done here. */
979
980	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage] ppv=%p\n", GCPhys, rc, pPage, ppv));
981	}
982	else
983	Log6(("PGMR3PhysTlbGCPhys2Ptr: GCPhys=%RGp rc=%Rrc pPage=%R[pgmpage]\n", GCPhys, rc, pPage));
984
985	/* else: handler catching all access, no pointer returned. */
986	}
987	else
988	rc = VERR_PGM_PHYS_TLB_UNASSIGNED;
989
990	PGM_UNLOCK(pVM);
991	return rc;
992	}
993
994
995
996	/*********************************************************************************************************************************
997	* RAM Range Management *
998	*********************************************************************************************************************************/
999
1000	/**
1001	* Given the range @a GCPhys thru @a GCPhysLast, find overlapping RAM range or
1002	* the correct insertion point.
1003	*
1004	* @returns Pointer to overlapping RAM range if found, NULL if not.
1005	* @param pVM The cross context VM structure.
1006	* @param GCPhys The address of the first byte in the range.
1007	* @param GCPhysLast The address of the last byte in the range.
1008	* @param pidxInsert Where to return the lookup table index to insert the
1009	* range at when returning NULL. Set to UINT32_MAX when
1010	* returning the pointer to an overlapping range.
1011	* @note Caller must own the PGM lock.
1012	*/
1013	static PPGMRAMRANGE pgmR3PhysRamRangeFindOverlapping(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, uint32_t *pidxInsert)
1014	{
1015	PGM_LOCK_ASSERT_OWNER(pVM);
1016	uint32_t iStart = 0;
1017	uint32_t iEnd = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1018	for (;;)
1019	{
1020	uint32_t idxLookup = iStart + (iEnd - iStart) / 2;
1021	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1022	if (GCPhysLast < GCPhysEntryFirst)
1023	{
1024	if (idxLookup > iStart)
1025	iEnd = idxLookup;
1026	else
1027	{
1028	*pidxInsert = idxLookup;
1029	return NULL;
1030	}
1031	}
1032	else
1033	{
1034	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1035	if (GCPhys > GCPhysEntryLast)
1036	{
1037	idxLookup += 1;
1038	if (idxLookup < iEnd)
1039	iStart = idxLookup;
1040	else
1041	{
1042	*pidxInsert = idxLookup;
1043	return NULL;
1044	}
1045	}
1046	else
1047	{
1048	/* overlap */
1049	Assert(GCPhysEntryLast > GCPhys && GCPhysEntryFirst < GCPhysLast);
1050	*pidxInsert = UINT32_MAX;
1051	return pVM->pgm.s.apRamRanges[PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup])];
1052	}
1053	}
1054	}
1055	}
1056
1057
1058	/**
1059	* Given the range @a GCPhys thru @a GCPhysLast, find the lookup table entry
1060	* that's overlapping it.
1061	*
1062	* @returns The lookup table index of the overlapping entry, UINT32_MAX if not
1063	* found.
1064	* @param pVM The cross context VM structure.
1065	* @param GCPhys The address of the first byte in the range.
1066	* @param GCPhysLast The address of the last byte in the range.
1067	* @note Caller must own the PGM lock.
1068	*/
1069	static uint32_t pgmR3PhysRamRangeFindOverlappingIndex(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast)
1070	{
1071	PGM_LOCK_ASSERT_OWNER(pVM);
1072	uint32_t iStart = 0;
1073	uint32_t iEnd = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1074	for (;;)
1075	{
1076	uint32_t idxLookup = iStart + (iEnd - iStart) / 2;
1077	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1078	if (GCPhysLast < GCPhysEntryFirst)
1079	{
1080	if (idxLookup > iStart)
1081	iEnd = idxLookup;
1082	else
1083	return UINT32_MAX;
1084	}
1085	else
1086	{
1087	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1088	if (GCPhys > GCPhysEntryLast)
1089	{
1090	idxLookup += 1;
1091	if (idxLookup < iEnd)
1092	iStart = idxLookup;
1093	else
1094	return UINT32_MAX;
1095	}
1096	else
1097	{
1098	/* overlap */
1099	Assert(GCPhysEntryLast > GCPhys && GCPhysEntryFirst < GCPhysLast);
1100	return idxLookup;
1101	}
1102	}
1103	}
1104	}
1105
1106
1107	/**
1108	* Insert @a pRam into the lookup table.
1109	*
1110	* @returns VBox status code.
1111	* @param pVM The cross context VM structure.
1112	* @param pRam The RAM range to insert into the lookup table.
1113	* @param pidxLookup Optional lookup table hint. This is updated.
1114	* @note Caller must own PGM lock.
1115	*/
1116	static int pgmR3PhysRamRangeInsertLookup(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, uint32_t *pidxLookup)
1117	{
1118	PGM_LOCK_ASSERT_OWNER(pVM);
1119	#ifdef DEBUG_bird
1120	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, true /fRamRelaxed/);
1121	#endif
1122	AssertMsg(pRam->pszDesc, ("%RGp-%RGp\n", pRam->GCPhys, pRam->GCPhysLast));
1123	AssertLogRelMsgReturn( pRam->GCPhys == NIL_RTGCPHYS
1124	&& pRam->GCPhysLast == NIL_RTGCPHYS,
1125	("GCPhys=%RGp; range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1126	GCPhys, pRam->GCPhys, pRam->cb, pRam->GCPhysLast, pRam->pszDesc),
1127	VERR_ALREADY_EXISTS);
1128	uint32_t const idRamRange = pRam->idRange;
1129	AssertReturn(pVM->pgm.s.apRamRanges[idRamRange] == pRam, VERR_INTERNAL_ERROR_2);
1130
1131	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INTERNAL_ERROR_3);
1132	RTGCPHYS const GCPhysLast = GCPhys + pRam->cb - 1U;
1133	AssertReturn(GCPhysLast > GCPhys, VERR_INTERNAL_ERROR_4);
1134	LogFlowFunc(("GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n", GCPhys, pRam->cb, GCPhysLast, idRamRange, pRam->pszDesc));
1135
1136	/*
1137	* Find the lookup table location if necessary.
1138	*/
1139	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1140	AssertLogRelMsgReturn(cLookupEntries + 1 < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), /* id=0 is unused, so < is correct. */
1141	("%#x\n", cLookupEntries), VERR_INTERNAL_ERROR_3);
1142
1143	uint32_t idxLookup = pidxLookup ? *pidxLookup : UINT32_MAX;
1144	if (cLookupEntries == 0)
1145	idxLookup = 0; /* special case: empty table */
1146	else
1147	{
1148	if ( idxLookup > cLookupEntries
1149	\|\| ( idxLookup != 0
1150	&& pVM->pgm.s.aRamRangeLookup[idxLookup - 1].GCPhysLast >= GCPhys)
1151	\|\| ( idxLookup < cLookupEntries
1152	&& PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]) < GCPhysLast))
1153	{
1154	PPGMRAMRANGE pOverlapping = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxLookup);
1155	AssertLogRelMsgReturn(!pOverlapping,
1156	("GCPhys=%RGp; GCPhysLast=%RGp %s - overlaps %RGp...%RGp %s\n",
1157	GCPhys, GCPhysLast, pRam->pszDesc,
1158	pOverlapping->GCPhys, pOverlapping->GCPhysLast, pOverlapping->pszDesc),
1159	VERR_PGM_RAM_CONFLICT);
1160	AssertLogRelMsgReturn(idxLookup <= cLookupEntries, ("%#x vs %#x\n", idxLookup, cLookupEntries), VERR_INTERNAL_ERROR_5);
1161	}
1162	/* else we've got a good hint. */
1163	}
1164
1165	/*
1166	* Do the actual job.
1167	*
1168	* The moving of existing table entries is done in a way that allows other
1169	* EMTs to perform concurrent lookups with the updating.
1170	*/
1171	bool const fUseAtomic = pVM->enmVMState != VMSTATE_CREATING
1172	&& pVM->cCpus > 1
1173	#ifdef RT_ARCH_AMD64
1174	&& g_CpumHostFeatures.s.fCmpXchg16b
1175	#endif
1176	;
1177
1178	/* Signal that we're modifying the lookup table: */
1179	uint32_t const idGeneration = (pVM->pgm.s.RamRangeUnion.idGeneration + 1) \| 1; /* paranoia^3 */
1180	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.idGeneration, idGeneration);
1181
1182	/* Update the RAM range entry. */
1183	pRam->GCPhys = GCPhys;
1184	pRam->GCPhysLast = GCPhysLast;
1185
1186	/* Do we need to shift any lookup table entries? */
1187	if (idxLookup != cLookupEntries)
1188	{
1189	/* We do. Make a copy of the final entry first. */
1190	uint32_t cToMove = cLookupEntries - idxLookup;
1191	PGMRAMRANGELOOKUPENTRY *pCur = &pVM->pgm.s.aRamRangeLookup[cLookupEntries];
1192	pCur->GCPhysFirstAndId = pCur[-1].GCPhysFirstAndId;
1193	pCur->GCPhysLast = pCur[-1].GCPhysLast;
1194
1195	/* Then increase the table size. This will ensure that anyone starting
1196	a search from here on should have consistent data. */
1197	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.cLookupEntries, cLookupEntries + 1);
1198
1199	/* Transfer the rest of the entries. */
1200	cToMove -= 1;
1201	if (cToMove > 0)
1202	{
1203	if (!fUseAtomic)
1204	do
1205	{
1206	pCur -= 1;
1207	pCur->GCPhysFirstAndId = pCur[-1].GCPhysFirstAndId;
1208	pCur->GCPhysLast = pCur[-1].GCPhysLast;
1209	cToMove -= 1;
1210	} while (cToMove > 0);
1211	else
1212	{
1213	#if RTASM_HAVE_WRITE_U128 >= 2
1214	do
1215	{
1216	pCur -= 1;
1217	ASMAtomicWriteU128U(&pCur->u128Volatile, pCur[-1].u128Normal);
1218	cToMove -= 1;
1219	} while (cToMove > 0);
1220
1221	#else
1222	uint64_t u64PrevLo = pCur[-1].u128Normal.s.Lo;
1223	uint64_t u64PrevHi = pCur[-1].u128Normal.s.Hi;
1224	do
1225	{
1226	pCur -= 1;
1227	uint64_t const u64CurLo = pCur[-1].u128Normal.s.Lo;
1228	uint64_t const u64CurHi = pCur[-1].u128Normal.s.Hi;
1229	uint128_t uOldIgn;
1230	AssertStmt(ASMAtomicCmpXchgU128v2(&pCur->u128Volatile.u, u64CurHi, u64CurLo, u64PrevHi, u64PrevLo, &uOldIgn),
1231	(pCur->u128Volatile.s.Lo = u64CurLo, pCur->u128Volatile.s.Hi = u64CurHi));
1232	u64PrevLo = u64CurLo;
1233	u64PrevHi = u64CurHi;
1234	cToMove -= 1;
1235	} while (cToMove > 0);
1236	#endif
1237	}
1238	}
1239	}
1240
1241	/*
1242	* Write the new entry.
1243	*/
1244	PGMRAMRANGELOOKUPENTRY *pInsert = &pVM->pgm.s.aRamRangeLookup[idxLookup];
1245	if (!fUseAtomic)
1246	{
1247	pInsert->GCPhysFirstAndId = idRamRange \| GCPhys;
1248	pInsert->GCPhysLast = GCPhysLast;
1249	}
1250	else
1251	{
1252	PGMRAMRANGELOOKUPENTRY NewEntry;
1253	NewEntry.GCPhysFirstAndId = idRamRange \| GCPhys;
1254	NewEntry.GCPhysLast = GCPhysLast;
1255	ASMAtomicWriteU128v2(&pInsert->u128Volatile.u, NewEntry.u128Normal.s.Hi, NewEntry.u128Normal.s.Lo);
1256	}
1257
1258	/*
1259	* Update the generation and count in one go, signaling the end of the updating.
1260	*/
1261	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT GenAndCount;
1262	GenAndCount.cLookupEntries = cLookupEntries + 1;
1263	GenAndCount.idGeneration = idGeneration + 1;
1264	ASMAtomicWriteU64(&pVM->pgm.s.RamRangeUnion.u64Combined, GenAndCount.u64Combined);
1265
1266	if (pidxLookup)
1267	*pidxLookup = idxLookup + 1;
1268
1269	#ifdef DEBUG_bird
1270	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
1271	#endif
1272	return VINF_SUCCESS;
1273	}
1274
1275
1276	/**
1277	* Removes @a pRam from the lookup table.
1278	*
1279	* @returns VBox status code.
1280	* @param pVM The cross context VM structure.
1281	* @param pRam The RAM range to insert into the lookup table.
1282	* @param pidxLookup Optional lookup table hint. This is updated.
1283	* @note Caller must own PGM lock.
1284	*/
1285	static int pgmR3PhysRamRangeRemoveLookup(PVM pVM, PPGMRAMRANGE pRam, uint32_t *pidxLookup)
1286	{
1287	PGM_LOCK_ASSERT_OWNER(pVM);
1288	AssertMsg(pRam->pszDesc, ("%RGp-%RGp\n", pRam->GCPhys, pRam->GCPhysLast));
1289
1290	RTGCPHYS const GCPhys = pRam->GCPhys;
1291	RTGCPHYS const GCPhysLast = pRam->GCPhysLast;
1292	AssertLogRelMsgReturn( GCPhys != NIL_RTGCPHYS
1293	\|\| GCPhysLast != NIL_RTGCPHYS,
1294	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n", GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1295	VERR_NOT_FOUND);
1296	AssertLogRelMsgReturn( GCPhys != NIL_RTGCPHYS
1297	&& GCPhysLast == GCPhys + pRam->cb - 1U
1298	&& (GCPhys & GUEST_PAGE_OFFSET_MASK) == 0
1299	&& (GCPhysLast & GUEST_PAGE_OFFSET_MASK) == GUEST_PAGE_OFFSET_MASK
1300	&& GCPhysLast > GCPhys,
1301	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n", GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1302	VERR_INTERNAL_ERROR_5);
1303	uint32_t const idRamRange = pRam->idRange;
1304	AssertReturn(pVM->pgm.s.apRamRanges[idRamRange] == pRam, VERR_INTERNAL_ERROR_4);
1305	LogFlowFunc(("GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n", GCPhys, pRam->cb, GCPhysLast, idRamRange, pRam->pszDesc));
1306
1307	/*
1308	* Find the lookup table location.
1309	*/
1310	uint32_t const cLookupEntries = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1311	AssertLogRelMsgReturn( cLookupEntries > 0
1312	&& cLookupEntries < RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup), /* id=0 is unused, so < is correct. */
1313	("%#x\n", cLookupEntries), VERR_INTERNAL_ERROR_3);
1314
1315	uint32_t idxLookup = pidxLookup ? *pidxLookup : UINT32_MAX;
1316	if ( idxLookup >= cLookupEntries
1317	\|\| pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast != GCPhysLast
1318	\|\| pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysFirstAndId != (GCPhys \| idRamRange))
1319	{
1320	uint32_t iStart = 0;
1321	uint32_t iEnd = cLookupEntries;
1322	for (;;)
1323	{
1324	idxLookup = iStart + (iEnd - iStart) / 2;
1325	RTGCPHYS const GCPhysEntryFirst = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1326	if (GCPhysLast < GCPhysEntryFirst)
1327	{
1328	AssertLogRelMsgReturn(idxLookup > iStart,
1329	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1330	GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1331	VERR_NOT_FOUND);
1332	iEnd = idxLookup;
1333	}
1334	else
1335	{
1336	RTGCPHYS const GCPhysEntryLast = pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast;
1337	if (GCPhys > GCPhysEntryLast)
1338	{
1339	idxLookup += 1;
1340	AssertLogRelMsgReturn(idxLookup < iEnd,
1341	("range: GCPhys=%RGp LB %RGp GCPhysLast=%RGp %s\n",
1342	GCPhys, pRam->cb, GCPhysLast, pRam->pszDesc),
1343	VERR_NOT_FOUND);
1344	iStart = idxLookup;
1345	}
1346	else
1347	{
1348	uint32_t const idEntry = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1349	AssertLogRelMsgReturn( GCPhysEntryFirst == GCPhys
1350	&& GCPhysEntryLast == GCPhysLast
1351	&& idEntry == idRamRange,
1352	("Found: %RGp..%RGp id=%#x; Wanted: GCPhys=%RGp LB %RGp GCPhysLast=%RGp id=%#x %s\n",
1353	GCPhysEntryFirst, GCPhysEntryLast, idEntry,
1354	GCPhys, pRam->cb, GCPhysLast, pRam->idRange, pRam->pszDesc),
1355	VERR_NOT_FOUND);
1356	break;
1357	}
1358	}
1359	}
1360	}
1361	/* else we've got a good hint. */
1362
1363	/*
1364	* Do the actual job.
1365	*
1366	* The moving of existing table entries is done in a way that allows other
1367	* EMTs to perform concurrent lookups with the updating.
1368	*/
1369	bool const fUseAtomic = pVM->enmVMState != VMSTATE_CREATING
1370	&& pVM->cCpus > 1
1371	#ifdef RT_ARCH_AMD64
1372	&& g_CpumHostFeatures.s.fCmpXchg16b
1373	#endif
1374	;
1375
1376	/* Signal that we're modifying the lookup table: */
1377	uint32_t const idGeneration = (pVM->pgm.s.RamRangeUnion.idGeneration + 1) \| 1; /* paranoia^3 */
1378	ASMAtomicWriteU32(&pVM->pgm.s.RamRangeUnion.idGeneration, idGeneration);
1379
1380	/* Do we need to shift any lookup table entries? (This is a lot simpler
1381	than insertion.) */
1382	if (idxLookup + 1U < cLookupEntries)
1383	{
1384	uint32_t cToMove = cLookupEntries - idxLookup - 1U;
1385	PGMRAMRANGELOOKUPENTRY *pCur = &pVM->pgm.s.aRamRangeLookup[idxLookup];
1386	if (!fUseAtomic)
1387	do
1388	{
1389	pCur->GCPhysFirstAndId = pCur[1].GCPhysFirstAndId;
1390	pCur->GCPhysLast = pCur[1].GCPhysLast;
1391	pCur += 1;
1392	cToMove -= 1;
1393	} while (cToMove > 0);
1394	else
1395	{
1396	#if RTASM_HAVE_WRITE_U128 >= 2
1397	do
1398	{
1399	ASMAtomicWriteU128U(&pCur->u128Volatile, pCur[1].u128Normal);
1400	pCur += 1;
1401	cToMove -= 1;
1402	} while (cToMove > 0);
1403
1404	#else
1405	uint64_t u64PrevLo = pCur->u128Normal.s.Lo;
1406	uint64_t u64PrevHi = pCur->u128Normal.s.Hi;
1407	do
1408	{
1409	uint64_t const u64CurLo = pCur[1].u128Normal.s.Lo;
1410	uint64_t const u64CurHi = pCur[1].u128Normal.s.Hi;
1411	uint128_t uOldIgn;
1412	AssertStmt(ASMAtomicCmpXchgU128v2(&pCur->u128Volatile.u, u64CurHi, u64CurLo, u64PrevHi, u64PrevLo, &uOldIgn),
1413	(pCur->u128Volatile.s.Lo = u64CurLo, pCur->u128Volatile.s.Hi = u64CurHi));
1414	u64PrevLo = u64CurLo;
1415	u64PrevHi = u64CurHi;
1416	pCur += 1;
1417	cToMove -= 1;
1418	} while (cToMove > 0);
1419	#endif
1420	}
1421	}
1422
1423	/* Update the RAM range entry to indicate that it is no longer mapped. */
1424	pRam->GCPhys = NIL_RTGCPHYS;
1425	pRam->GCPhysLast = NIL_RTGCPHYS;
1426
1427	/*
1428	* Update the generation and count in one go, signaling the end of the updating.
1429	*/
1430	PGM::PGMRAMRANGEGENANDLOOKUPCOUNT GenAndCount;
1431	GenAndCount.cLookupEntries = cLookupEntries - 1;
1432	GenAndCount.idGeneration = idGeneration + 1;
1433	ASMAtomicWriteU64(&pVM->pgm.s.RamRangeUnion.u64Combined, GenAndCount.u64Combined);
1434
1435	if (pidxLookup)
1436	*pidxLookup = idxLookup + 1;
1437
1438	return VINF_SUCCESS;
1439	}
1440
1441
1442	/**
1443	* Gets the number of ram ranges.
1444	*
1445	* @returns Number of ram ranges. Returns UINT32_MAX if @a pVM is invalid.
1446	* @param pVM The cross context VM structure.
1447	*/
1448	VMMR3DECL(uint32_t) PGMR3PhysGetRamRangeCount(PVM pVM)
1449	{
1450	VM_ASSERT_VALID_EXT_RETURN(pVM, UINT32_MAX);
1451
1452	PGM_LOCK_VOID(pVM);
1453	uint32_t const cRamRanges = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
1454	PGM_UNLOCK(pVM);
1455	return cRamRanges;
1456	}
1457
1458
1459	/**
1460	* Get information about a range.
1461	*
1462	* @returns VINF_SUCCESS or VERR_OUT_OF_RANGE.
1463	* @param pVM The cross context VM structure.
1464	* @param iRange The ordinal of the range.
1465	* @param pGCPhysStart Where to return the start of the range. Optional.
1466	* @param pGCPhysLast Where to return the address of the last byte in the
1467	* range. Optional.
1468	* @param ppszDesc Where to return the range description. Optional.
1469	* @param pfIsMmio Where to indicate that this is a pure MMIO range.
1470	* Optional.
1471	*/
1472	VMMR3DECL(int) PGMR3PhysGetRange(PVM pVM, uint32_t iRange, PRTGCPHYS pGCPhysStart, PRTGCPHYS pGCPhysLast,
1473	const char *ppszDesc, bool pfIsMmio)
1474	{
1475	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1476
1477	PGM_LOCK_VOID(pVM);
1478	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
1479	if (iRange < cLookupEntries)
1480	{
1481	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[iRange]);
1482	Assert(idRamRange && idRamRange <= pVM->pgm.s.idRamRangeMax);
1483	PGMRAMRANGE const * const pRamRange = pVM->pgm.s.apRamRanges[idRamRange];
1484	AssertPtr(pRamRange);
1485
1486	if (pGCPhysStart)
1487	*pGCPhysStart = pRamRange->GCPhys;
1488	if (pGCPhysLast)
1489	*pGCPhysLast = pRamRange->GCPhysLast;
1490	if (ppszDesc)
1491	*ppszDesc = pRamRange->pszDesc;
1492	if (pfIsMmio)
1493	*pfIsMmio = !!(pRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO);
1494
1495	PGM_UNLOCK(pVM);
1496	return VINF_SUCCESS;
1497	}
1498	PGM_UNLOCK(pVM);
1499	return VERR_OUT_OF_RANGE;
1500	}
1501
1502
1503	/*********************************************************************************************************************************
1504	* RAM *
1505	*********************************************************************************************************************************/
1506
1507	/**
1508	* Frees the specified RAM page and replaces it with the ZERO page.
1509	*
1510	* This is used by ballooning, remapping MMIO2, RAM reset and state loading.
1511	*
1512	* @param pVM The cross context VM structure.
1513	* @param pReq Pointer to the request. This is NULL when doing a
1514	* bulk free in NEM memory mode.
1515	* @param pcPendingPages Where the number of pages waiting to be freed are
1516	* kept. This will normally be incremented. This is
1517	* NULL when doing a bulk free in NEM memory mode.
1518	* @param pPage Pointer to the page structure.
1519	* @param GCPhys The guest physical address of the page, if applicable.
1520	* @param enmNewType New page type for NEM notification, since several
1521	* callers will change the type upon successful return.
1522	*
1523	* @remarks The caller must own the PGM lock.
1524	*/
1525	int pgmPhysFreePage(PVM pVM, PGMMFREEPAGESREQ pReq, uint32_t *pcPendingPages, PPGMPAGE pPage, RTGCPHYS GCPhys,
1526	PGMPAGETYPE enmNewType)
1527	{
1528	/*
1529	* Assert sanity.
1530	*/
1531	PGM_LOCK_ASSERT_OWNER(pVM);
1532	if (RT_UNLIKELY( PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM
1533	&& PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_ROM_SHADOW))
1534	{
1535	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1536	return VMSetError(pVM, VERR_PGM_PHYS_NOT_RAM, RT_SRC_POS, "GCPhys=%RGp type=%d", GCPhys, PGM_PAGE_GET_TYPE(pPage));
1537	}
1538
1539	/** @todo What about ballooning of large pages??! */
1540	Assert( PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE
1541	&& PGM_PAGE_GET_PDE_TYPE(pPage) != PGM_PAGE_PDE_TYPE_PDE_DISABLED);
1542
1543	if ( PGM_PAGE_IS_ZERO(pPage)
1544	\|\| PGM_PAGE_IS_BALLOONED(pPage))
1545	return VINF_SUCCESS;
1546
1547	const uint32_t idPage = PGM_PAGE_GET_PAGEID(pPage);
1548	Log3(("pgmPhysFreePage: idPage=%#x GCPhys=%RGp pPage=%R[pgmpage]\n", idPage, GCPhys, pPage));
1549	if (RT_UNLIKELY(!PGM_IS_IN_NEM_MODE(pVM)
1550	? idPage == NIL_GMM_PAGEID
1551	\|\| idPage > GMM_PAGEID_LAST
1552	\|\| PGM_PAGE_GET_CHUNKID(pPage) == NIL_GMM_CHUNKID
1553	: idPage != NIL_GMM_PAGEID))
1554	{
1555	AssertMsgFailed(("GCPhys=%RGp pPage=%R[pgmpage]\n", GCPhys, pPage));
1556	return VMSetError(pVM, VERR_PGM_PHYS_INVALID_PAGE_ID, RT_SRC_POS, "GCPhys=%RGp idPage=%#x", GCPhys, pPage);
1557	}
1558	#ifdef VBOX_WITH_NATIVE_NEM
1559	const RTHCPHYS HCPhysPrev = PGM_PAGE_GET_HCPHYS(pPage);
1560	#endif
1561
1562	/* update page count stats. */
1563	if (PGM_PAGE_IS_SHARED(pPage))
1564	pVM->pgm.s.cSharedPages--;
1565	else
1566	pVM->pgm.s.cPrivatePages--;
1567	pVM->pgm.s.cZeroPages++;
1568
1569	/* Deal with write monitored pages. */
1570	if (PGM_PAGE_GET_STATE(pPage) == PGM_PAGE_STATE_WRITE_MONITORED)
1571	{
1572	PGM_PAGE_SET_WRITTEN_TO(pVM, pPage);
1573	pVM->pgm.s.cWrittenToPages++;
1574	}
1575	PGM_PAGE_CLEAR_CODE_PAGE(pVM, pPage); /* No callback needed, IEMTlbInvalidateAllPhysicalAllCpus is called below. */
1576
1577	/*
1578	* pPage = ZERO page.
1579	*/
1580	PGM_PAGE_SET_HCPHYS(pVM, pPage, pVM->pgm.s.HCPhysZeroPg);
1581	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
1582	PGM_PAGE_SET_PAGEID(pVM, pPage, NIL_GMM_PAGEID);
1583	PGM_PAGE_SET_PDE_TYPE(pVM, pPage, PGM_PAGE_PDE_TYPE_DONTCARE);
1584	PGM_PAGE_SET_PTE_INDEX(pVM, pPage, 0);
1585	PGM_PAGE_SET_TRACKING(pVM, pPage, 0);
1586
1587	/* Flush physical page map TLB entry. */
1588	pgmPhysInvalidatePageMapTLBEntry(pVM, GCPhys);
1589	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_FREED); /// @todo move to the perform step.
1590
1591	#ifdef VBOX_WITH_PGM_NEM_MODE
1592	/*
1593	* Skip the rest if we're doing a bulk free in NEM memory mode.
1594	*/
1595	if (!pReq)
1596	return VINF_SUCCESS;
1597	AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
1598	#endif
1599
1600	#ifdef VBOX_WITH_NATIVE_NEM
1601	/* Notify NEM. */
1602	/** @todo Remove this one? */
1603	if (VM_IS_NEM_ENABLED(pVM))
1604	{
1605	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pPage);
1606	NEMHCNotifyPhysPageChanged(pVM, GCPhys, HCPhysPrev, pVM->pgm.s.HCPhysZeroPg, pVM->pgm.s.abZeroPg,
1607	pgmPhysPageCalcNemProtection(pPage, enmNewType), enmNewType, &u2State);
1608	PGM_PAGE_SET_NEM_STATE(pPage, u2State);
1609	}
1610	#else
1611	RT_NOREF(enmNewType);
1612	#endif
1613
1614	/*
1615	* Make sure it's not in the handy page array.
1616	*/
1617	for (uint32_t i = pVM->pgm.s.cHandyPages; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
1618	{
1619	if (pVM->pgm.s.aHandyPages[i].idPage == idPage)
1620	{
1621	pVM->pgm.s.aHandyPages[i].HCPhysGCPhys = NIL_GMMPAGEDESC_PHYS;
1622	pVM->pgm.s.aHandyPages[i].fZeroed = false;
1623	pVM->pgm.s.aHandyPages[i].idPage = NIL_GMM_PAGEID;
1624	break;
1625	}
1626	if (pVM->pgm.s.aHandyPages[i].idSharedPage == idPage)
1627	{
1628	pVM->pgm.s.aHandyPages[i].idSharedPage = NIL_GMM_PAGEID;
1629	break;
1630	}
1631	}
1632
1633	/*
1634	* Push it onto the page array.
1635	*/
1636	uint32_t iPage = *pcPendingPages;
1637	Assert(iPage < PGMPHYS_FREE_PAGE_BATCH_SIZE);
1638	*pcPendingPages += 1;
1639
1640	pReq->aPages[iPage].idPage = idPage;
1641
1642	if (iPage + 1 < PGMPHYS_FREE_PAGE_BATCH_SIZE)
1643	return VINF_SUCCESS;
1644
1645	/*
1646	* Flush the pages.
1647	*/
1648	int rc = GMMR3FreePagesPerform(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE);
1649	if (RT_SUCCESS(rc))
1650	{
1651	GMMR3FreePagesRePrep(pVM, pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1652	*pcPendingPages = 0;
1653	}
1654	return rc;
1655	}
1656
1657
1658	/**
1659	* Frees a range of pages, replacing them with ZERO pages of the specified type.
1660	*
1661	* @returns VBox status code.
1662	* @param pVM The cross context VM structure.
1663	* @param pRam The RAM range in which the pages resides.
1664	* @param GCPhys The address of the first page.
1665	* @param GCPhysLast The address of the last page.
1666	* @param pvMmio2 Pointer to the ring-3 mapping of any MMIO2 memory that
1667	* will replace the pages we're freeing up.
1668	*/
1669	static int pgmR3PhysFreePageRange(PVM pVM, PPGMRAMRANGE pRam, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast, void *pvMmio2)
1670	{
1671	PGM_LOCK_ASSERT_OWNER(pVM);
1672
1673	#ifdef VBOX_WITH_PGM_NEM_MODE
1674	/*
1675	* In simplified memory mode we don't actually free the memory,
1676	* we just unmap it and let NEM do any unlocking of it.
1677	*/
1678	if (pVM->pgm.s.fNemMode)
1679	{
1680	Assert(VM_IS_NEM_ENABLED(pVM) \|\| VM_IS_EXEC_ENGINE_IEM(pVM));
1681	uint8_t u2State = 0; /* (We don't support UINT8_MAX here.) */
1682	if (VM_IS_NEM_ENABLED(pVM))
1683	{
1684	uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1685	int rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify,
1686	pRam->pbR3 ? pRam->pbR3 + GCPhys - pRam->GCPhys : NULL,
1687	pvMmio2, &u2State, NULL /puNemRange/);
1688	AssertLogRelRCReturn(rc, rc);
1689	}
1690
1691	/* Iterate the pages. */
1692	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1693	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1694	while (cPagesLeft-- > 0)
1695	{
1696	int rc = pgmPhysFreePage(pVM, NULL, NULL, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1697	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1698
1699	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1700	PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1701
1702	GCPhys += GUEST_PAGE_SIZE;
1703	pPageDst++;
1704	}
1705	return VINF_SUCCESS;
1706	}
1707	#else /* !VBOX_WITH_PGM_NEM_MODE */
1708	RT_NOREF(pvMmio2);
1709	#endif /* !VBOX_WITH_PGM_NEM_MODE */
1710
1711	/*
1712	* Regular mode.
1713	*/
1714	/* Prepare. */
1715	uint32_t cPendingPages = 0;
1716	PGMMFREEPAGESREQ pReq;
1717	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
1718	AssertLogRelRCReturn(rc, rc);
1719
1720	#ifdef VBOX_WITH_NATIVE_NEM
1721	/* Tell NEM up-front. */
1722	uint8_t u2State = UINT8_MAX;
1723	if (VM_IS_NEM_ENABLED(pVM))
1724	{
1725	uint32_t const fNemNotify = (pvMmio2 ? NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2 : 0) \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE;
1726	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, GCPhysLast - GCPhys + 1, fNemNotify, NULL, pvMmio2,
1727	&u2State, NULL /puNemRange/);
1728	AssertLogRelRCReturnStmt(rc, GMMR3FreePagesCleanup(pReq), rc);
1729	}
1730	#endif
1731
1732	/* Iterate the pages. */
1733	PPGMPAGE pPageDst = &pRam->aPages[(GCPhys - pRam->GCPhys) >> GUEST_PAGE_SHIFT];
1734	uint32_t cPagesLeft = ((GCPhysLast - GCPhys) >> GUEST_PAGE_SHIFT) + 1;
1735	while (cPagesLeft-- > 0)
1736	{
1737	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPageDst, GCPhys, PGMPAGETYPE_MMIO);
1738	AssertLogRelRCReturn(rc, rc); /* We're done for if this goes wrong. */
1739
1740	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO);
1741	#ifdef VBOX_WITH_NATIVE_NEM
1742	if (u2State != UINT8_MAX)
1743	PGM_PAGE_SET_NEM_STATE(pPageDst, u2State);
1744	#endif
1745
1746	GCPhys += GUEST_PAGE_SIZE;
1747	pPageDst++;
1748	}
1749
1750	/* Finish pending and cleanup. */
1751	if (cPendingPages)
1752	{
1753	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
1754	AssertLogRelRCReturn(rc, rc);
1755	}
1756	GMMR3FreePagesCleanup(pReq);
1757
1758	return rc;
1759	}
1760
1761
1762	/**
1763	* Wrapper around VMMR0_DO_PGM_PHYS_ALLOCATE_RAM_RANGE.
1764	*/
1765	static int pgmR3PhysAllocateRamRange(PVM pVM, PVMCPU pVCpu, uint32_t cGuestPages, uint32_t fFlags, PPGMRAMRANGE *ppRamRange)
1766	{
1767	int rc;
1768	PGMPHYSALLOCATERAMRANGEREQ AllocRangeReq;
1769	AllocRangeReq.idNewRange = UINT32_MAX / 4;
1770	if (SUPR3IsDriverless())
1771	rc = pgmPhysRamRangeAllocCommon(pVM, cGuestPages, fFlags, &AllocRangeReq.idNewRange);
1772	else
1773	{
1774	AllocRangeReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
1775	AllocRangeReq.Hdr.cbReq = sizeof(AllocRangeReq);
1776	AllocRangeReq.cbGuestPage = GUEST_PAGE_SIZE;
1777	AllocRangeReq.cGuestPages = cGuestPages;
1778	AllocRangeReq.fFlags = fFlags;
1779	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_ALLOCATE_RAM_RANGE, 0 /u64Arg/, &AllocRangeReq.Hdr);
1780	}
1781	if (RT_SUCCESS(rc))
1782	{
1783	Assert(AllocRangeReq.idNewRange != 0);
1784	Assert(AllocRangeReq.idNewRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
1785	AssertPtr(pVM->pgm.s.apRamRanges[AllocRangeReq.idNewRange]);
1786	*ppRamRange = pVM->pgm.s.apRamRanges[AllocRangeReq.idNewRange];
1787	return VINF_SUCCESS;
1788	}
1789
1790	*ppRamRange = NULL;
1791	return rc;
1792	}
1793
1794
1795	/**
1796	* PGMR3PhysRegisterRam worker that initializes and links a RAM range.
1797	*
1798	* In NEM mode, this will allocate the pages backing the RAM range and this may
1799	* fail. NEM registration may also fail. (In regular HM mode it won't fail.)
1800	*
1801	* @returns VBox status code.
1802	* @param pVM The cross context VM structure.
1803	* @param pNew The new RAM range.
1804	* @param GCPhys The address of the RAM range.
1805	* @param GCPhysLast The last address of the RAM range.
1806	* @param pszDesc The description.
1807	* @param pidxLookup The lookup table insertion point.
1808	*/
1809	static int pgmR3PhysInitAndLinkRamRange(PVM pVM, PPGMRAMRANGE pNew, RTGCPHYS GCPhys, RTGCPHYS GCPhysLast,
1810	const char pszDesc, uint32_t pidxLookup)
1811	{
1812	/*
1813	* Initialize the range.
1814	*/
1815	Assert(pNew->cb == GCPhysLast - GCPhys + 1U);
1816	pNew->pszDesc = pszDesc;
1817	pNew->uNemRange = UINT32_MAX;
1818	pNew->pbR3 = NULL;
1819	pNew->paLSPages = NULL;
1820
1821	uint32_t const cPages = pNew->cb >> GUEST_PAGE_SHIFT;
1822	#ifdef VBOX_WITH_PGM_NEM_MODE
1823	if (!pVM->pgm.s.fNemMode)
1824	#endif
1825	{
1826	RTGCPHYS iPage = cPages;
1827	while (iPage-- > 0)
1828	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_RAM);
1829
1830	/* Update the page count stats. */
1831	pVM->pgm.s.cZeroPages += cPages;
1832	pVM->pgm.s.cAllPages += cPages;
1833	}
1834	#ifdef VBOX_WITH_PGM_NEM_MODE
1835	else
1836	{
1837	int rc = SUPR3PageAlloc(RT_ALIGN_Z(pNew->cb, HOST_PAGE_SIZE) >> HOST_PAGE_SHIFT,
1838	pVM->pgm.s.fUseLargePages ? SUP_PAGE_ALLOC_F_LARGE_PAGES : 0, (void **)&pNew->pbR3);
1839	if (RT_FAILURE(rc))
1840	return rc;
1841
1842	RTGCPHYS iPage = cPages;
1843	while (iPage-- > 0)
1844	PGM_PAGE_INIT(&pNew->aPages[iPage], UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
1845	PGMPAGETYPE_RAM, PGM_PAGE_STATE_ALLOCATED);
1846
1847	/* Update the page count stats. */
1848	pVM->pgm.s.cPrivatePages += cPages;
1849	pVM->pgm.s.cAllPages += cPages;
1850	}
1851	#endif
1852
1853	/*
1854	* Insert it into the lookup table.
1855	*/
1856	int rc = pgmR3PhysRamRangeInsertLookup(pVM, pNew, GCPhys, pidxLookup);
1857	AssertRCReturn(rc, rc);
1858
1859	#ifdef VBOX_WITH_NATIVE_NEM
1860	/*
1861	* Notify NEM now that it has been linked.
1862	*
1863	* As above, it is assumed that on failure the VM creation will fail, so
1864	* no extra cleanup is needed here.
1865	*/
1866	if (VM_IS_NEM_ENABLED(pVM))
1867	{
1868	uint8_t u2State = UINT8_MAX;
1869	rc = NEMR3NotifyPhysRamRegister(pVM, GCPhys, pNew->cb, pNew->pbR3, &u2State, &pNew->uNemRange);
1870	if (RT_SUCCESS(rc) && u2State != UINT8_MAX)
1871	pgmPhysSetNemStateForPages(&pNew->aPages[0], cPages, u2State);
1872	return rc;
1873	}
1874	#endif
1875	return VINF_SUCCESS;
1876	}
1877
1878
1879	/**
1880	* Worker for PGMR3PhysRegisterRam called with the PGM lock.
1881	*
1882	* The caller releases the lock.
1883	*/
1884	static int pgmR3PhysRegisterRamWorker(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc,
1885	uint32_t const cRamRanges, RTGCPHYS const GCPhysLast)
1886	{
1887	#ifdef VBOX_STRICT
1888	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
1889	#endif
1890
1891	/*
1892	* Check that we've got enough free RAM ranges.
1893	*/
1894	AssertLogRelMsgReturn((uint64_t)pVM->pgm.s.idRamRangeMax + cRamRanges + 1 <= RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup),
1895	("idRamRangeMax=%#RX32 vs GCPhys=%RGp cb=%RGp / %#RX32 ranges (%s)\n",
1896	pVM->pgm.s.idRamRangeMax, GCPhys, cb, cRamRanges, pszDesc),
1897	VERR_PGM_TOO_MANY_RAM_RANGES);
1898
1899	/*
1900	* Check for conflicts via the lookup table. We search it backwards,
1901	* assuming that memory is added in ascending order by address.
1902	*/
1903	uint32_t idxLookup = pVM->pgm.s.RamRangeUnion.cLookupEntries;
1904	while (idxLookup)
1905	{
1906	if (GCPhys > pVM->pgm.s.aRamRangeLookup[idxLookup - 1].GCPhysLast)
1907	break;
1908	idxLookup--;
1909	RTGCPHYS const GCPhysCur = PGMRAMRANGELOOKUPENTRY_GET_FIRST(pVM->pgm.s.aRamRangeLookup[idxLookup]);
1910	AssertLogRelMsgReturn( GCPhysLast < GCPhysCur
1911	\|\| GCPhys > pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast,
1912	("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
1913	GCPhys, GCPhysLast, pszDesc, GCPhysCur, pVM->pgm.s.aRamRangeLookup[idxLookup].GCPhysLast,
1914	pVM->pgm.s.apRamRanges[PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup])]->pszDesc),
1915	VERR_PGM_RAM_CONFLICT);
1916	}
1917
1918	/*
1919	* Register it with GMM (the API bitches).
1920	*/
1921	const RTGCPHYS cPages = cb >> GUEST_PAGE_SHIFT;
1922	int rc = MMR3IncreaseBaseReservation(pVM, cPages);
1923	if (RT_FAILURE(rc))
1924	return rc;
1925
1926	/*
1927	* Create the required chunks.
1928	*/
1929	RTGCPHYS cPagesLeft = cPages;
1930	RTGCPHYS GCPhysChunk = GCPhys;
1931	uint32_t idxChunk = 0;
1932	while (cPagesLeft > 0)
1933	{
1934	uint32_t cPagesInChunk = cPagesLeft;
1935	if (cPagesInChunk > PGM_MAX_PAGES_PER_RAM_RANGE)
1936	cPagesInChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
1937
1938	const char *pszDescChunk = idxChunk == 0
1939	? pszDesc
1940	: MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s (#%u)", pszDesc, idxChunk + 1);
1941	AssertReturn(pszDescChunk, VERR_NO_MEMORY);
1942
1943	/*
1944	* Allocate a RAM range.
1945	*/
1946	PPGMRAMRANGE pNew = NULL;
1947	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cPagesInChunk, 0 /fFlags/, &pNew);
1948	AssertLogRelMsgReturn(RT_SUCCESS(rc),
1949	("pgmR3PhysAllocateRamRange failed: GCPhysChunk=%RGp cPagesInChunk=%#RX32 (%s): %Rrc\n",
1950	GCPhysChunk, cPagesInChunk, pszDescChunk, rc),
1951	rc);
1952
1953	/*
1954	* Ok, init and link the range.
1955	*/
1956	rc = pgmR3PhysInitAndLinkRamRange(pVM, pNew, GCPhysChunk,
1957	GCPhysChunk + ((RTGCPHYS)cPagesInChunk << GUEST_PAGE_SHIFT) - 1U,
1958	pszDescChunk, &idxLookup);
1959	AssertLogRelMsgReturn(RT_SUCCESS(rc),
1960	("pgmR3PhysInitAndLinkRamRange failed: GCPhysChunk=%RGp cPagesInChunk=%#RX32 (%s): %Rrc\n",
1961	GCPhysChunk, cPagesInChunk, pszDescChunk, rc),
1962	rc);
1963
1964	/* advance */
1965	GCPhysChunk += (RTGCPHYS)cPagesInChunk << GUEST_PAGE_SHIFT;
1966	cPagesLeft -= cPagesInChunk;
1967	idxChunk++;
1968	}
1969
1970	return rc;
1971	}
1972
1973
1974	/**
1975	* Sets up a range RAM.
1976	*
1977	* This will check for conflicting registrations, make a resource reservation
1978	* for the memory (with GMM), and setup the per-page tracking structures
1979	* (PGMPAGE).
1980	*
1981	* @returns VBox status code.
1982	* @param pVM The cross context VM structure.
1983	* @param GCPhys The physical address of the RAM.
1984	* @param cb The size of the RAM.
1985	* @param pszDesc The description - not copied, so, don't free or change it.
1986	*/
1987	VMMR3DECL(int) PGMR3PhysRegisterRam(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, const char *pszDesc)
1988	{
1989	/*
1990	* Validate input.
1991	*/
1992	Log(("PGMR3PhysRegisterRam: GCPhys=%RGp cb=%RGp pszDesc=%s\n", GCPhys, cb, pszDesc));
1993	AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
1994	AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
1995	AssertReturn(cb > 0, VERR_INVALID_PARAMETER);
1996	RTGCPHYS const GCPhysLast = GCPhys + (cb - 1);
1997	AssertMsgReturn(GCPhysLast > GCPhys, ("The range wraps! GCPhys=%RGp cb=%RGp\n", GCPhys, cb), VERR_INVALID_PARAMETER);
1998	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
1999	PVMCPU const pVCpu = VMMGetCpu(pVM);
2000	AssertReturn(pVCpu, VERR_VM_THREAD_NOT_EMT);
2001	AssertReturn(pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
2002
2003	/*
2004	* Calculate the number of RAM ranges required.
2005	* See also pgmPhysMmio2CalcChunkCount.
2006	*/
2007	uint32_t const cPagesPerChunk = PGM_MAX_PAGES_PER_RAM_RANGE;
2008	uint32_t const cRamRanges = (uint32_t)(((cb >> GUEST_PAGE_SHIFT) + cPagesPerChunk - 1) / cPagesPerChunk);
2009	AssertLogRelMsgReturn(cRamRanges * (RTGCPHYS)cPagesPerChunk * GUEST_PAGE_SIZE >= cb,
2010	("cb=%RGp cRamRanges=%#RX32 cPagesPerChunk=%#RX32\n", cb, cRamRanges, cPagesPerChunk),
2011	VERR_OUT_OF_RANGE);
2012
2013	PGM_LOCK_VOID(pVM);
2014
2015	int rc = pgmR3PhysRegisterRamWorker(pVM, pVCpu, GCPhys, cb, pszDesc, cRamRanges, GCPhysLast);
2016	#ifdef VBOX_STRICT
2017	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2018	#endif
2019
2020	PGM_UNLOCK(pVM);
2021	return rc;
2022	}
2023
2024
2025	/**
2026	* Worker called by PGMR3InitFinalize if we're configured to pre-allocate RAM.
2027	*
2028	* We do this late in the init process so that all the ROM and MMIO ranges have
2029	* been registered already and we don't go wasting memory on them.
2030	*
2031	* @returns VBox status code.
2032	*
2033	* @param pVM The cross context VM structure.
2034	*/
2035	int pgmR3PhysRamPreAllocate(PVM pVM)
2036	{
2037	Assert(pVM->pgm.s.fRamPreAlloc);
2038	Log(("pgmR3PhysRamPreAllocate: enter\n"));
2039	#ifdef VBOX_WITH_PGM_NEM_MODE
2040	AssertLogRelReturn(!pVM->pgm.s.fNemMode, VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
2041	#endif
2042
2043	/*
2044	* Walk the RAM ranges and allocate all RAM pages, halt at
2045	* the first allocation error.
2046	*/
2047	uint64_t cPages = 0;
2048	uint64_t NanoTS = RTTimeNanoTS();
2049	PGM_LOCK_VOID(pVM);
2050	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
2051	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
2052	{
2053	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2054	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
2055	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2056	AssertContinue(pRam);
2057
2058	PPGMPAGE pPage = &pRam->aPages[0];
2059	RTGCPHYS GCPhys = pRam->GCPhys;
2060	uint32_t cLeft = pRam->cb >> GUEST_PAGE_SHIFT;
2061	while (cLeft-- > 0)
2062	{
2063	if (PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM)
2064	{
2065	switch (PGM_PAGE_GET_STATE(pPage))
2066	{
2067	case PGM_PAGE_STATE_ZERO:
2068	{
2069	int rc = pgmPhysAllocPage(pVM, pPage, GCPhys);
2070	if (RT_FAILURE(rc))
2071	{
2072	LogRel(("PGM: RAM Pre-allocation failed at %RGp (in %s) with rc=%Rrc\n", GCPhys, pRam->pszDesc, rc));
2073	PGM_UNLOCK(pVM);
2074	return rc;
2075	}
2076	cPages++;
2077	break;
2078	}
2079
2080	case PGM_PAGE_STATE_BALLOONED:
2081	case PGM_PAGE_STATE_ALLOCATED:
2082	case PGM_PAGE_STATE_WRITE_MONITORED:
2083	case PGM_PAGE_STATE_SHARED:
2084	/* nothing to do here. */
2085	break;
2086	}
2087	}
2088
2089	/* next */
2090	pPage++;
2091	GCPhys += GUEST_PAGE_SIZE;
2092	}
2093	}
2094	PGM_UNLOCK(pVM);
2095	NanoTS = RTTimeNanoTS() - NanoTS;
2096
2097	LogRel(("PGM: Pre-allocated %llu pages in %llu ms\n", cPages, NanoTS / 1000000));
2098	Log(("pgmR3PhysRamPreAllocate: returns VINF_SUCCESS\n"));
2099	return VINF_SUCCESS;
2100	}
2101
2102
2103	/**
2104	* Checks shared page checksums.
2105	*
2106	* @param pVM The cross context VM structure.
2107	*/
2108	void pgmR3PhysAssertSharedPageChecksums(PVM pVM)
2109	{
2110	#ifdef VBOX_STRICT
2111	PGM_LOCK_VOID(pVM);
2112
2113	if (pVM->pgm.s.cSharedPages > 0)
2114	{
2115	/*
2116	* Walk the ram ranges.
2117	*/
2118	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
2119	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
2120	{
2121	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2122	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
2123	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2124	AssertContinue(pRam);
2125
2126	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2127	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb,
2128	("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2129
2130	while (iPage-- > 0)
2131	{
2132	PPGMPAGE pPage = &pRam->aPages[iPage];
2133	if (PGM_PAGE_IS_SHARED(pPage))
2134	{
2135	uint32_t u32Checksum = pPage->s.u2Unused0/* \| ((uint32_t)pPage->s.u2Unused1 << 8)*/;
2136	if (!u32Checksum)
2137	{
2138	RTGCPHYS GCPhysPage = pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT);
2139	void const *pvPage;
2140	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhysPage, &pvPage);
2141	if (RT_SUCCESS(rc))
2142	{
2143	uint32_t u32Checksum2 = RTCrc32(pvPage, GUEST_PAGE_SIZE);
2144	# if 0
2145	AssertMsg((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum, ("GCPhysPage=%RGp\n", GCPhysPage));
2146	# else
2147	if ((u32Checksum2 & /UINT32_C(0x00000303)/ 0x3) == u32Checksum)
2148	LogFlow(("shpg %#x @ %RGp %#x [OK]\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2149	else
2150	AssertMsgFailed(("shpg %#x @ %RGp %#x\n", PGM_PAGE_GET_PAGEID(pPage), GCPhysPage, u32Checksum2));
2151	# endif
2152	}
2153	else
2154	AssertRC(rc);
2155	}
2156	}
2157
2158	} /* for each page */
2159
2160	} /* for each ram range */
2161	}
2162
2163	PGM_UNLOCK(pVM);
2164	#endif /* VBOX_STRICT */
2165	NOREF(pVM);
2166	}
2167
2168
2169	/**
2170	* Resets the physical memory state.
2171	*
2172	* ASSUMES that the caller owns the PGM lock.
2173	*
2174	* @returns VBox status code.
2175	* @param pVM The cross context VM structure.
2176	*/
2177	int pgmR3PhysRamReset(PVM pVM)
2178	{
2179	PGM_LOCK_ASSERT_OWNER(pVM);
2180
2181	/* Reset the memory balloon. */
2182	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2183	AssertRC(rc);
2184
2185	#ifdef VBOX_WITH_PAGE_SHARING
2186	/* Clear all registered shared modules. */
2187	pgmR3PhysAssertSharedPageChecksums(pVM);
2188	rc = GMMR3ResetSharedModules(pVM);
2189	AssertRC(rc);
2190	#endif
2191	/* Reset counters. */
2192	pVM->pgm.s.cReusedSharedPages = 0;
2193	pVM->pgm.s.cBalloonedPages = 0;
2194
2195	return VINF_SUCCESS;
2196	}
2197
2198
2199	/**
2200	* Resets (zeros) the RAM after all devices and components have been reset.
2201	*
2202	* ASSUMES that the caller owns the PGM lock.
2203	*
2204	* @returns VBox status code.
2205	* @param pVM The cross context VM structure.
2206	*/
2207	int pgmR3PhysRamZeroAll(PVM pVM)
2208	{
2209	PGM_LOCK_ASSERT_OWNER(pVM);
2210
2211	/*
2212	* We batch up pages that should be freed instead of calling GMM for
2213	* each and every one of them.
2214	*/
2215	uint32_t cPendingPages = 0;
2216	PGMMFREEPAGESREQ pReq;
2217	int rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2218	AssertLogRelRCReturn(rc, rc);
2219
2220	/*
2221	* Walk the ram ranges.
2222	*/
2223	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
2224	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
2225	{
2226	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2227	Assert(pRam \|\| idRamRange == 0);
2228	if (!pRam) continue;
2229	Assert(pRam->idRange == idRamRange);
2230
2231	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2232	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2233
2234	if ( !pVM->pgm.s.fRamPreAlloc
2235	#ifdef VBOX_WITH_PGM_NEM_MODE
2236	&& !pVM->pgm.s.fNemMode
2237	#endif
2238	&& pVM->pgm.s.fZeroRamPagesOnReset)
2239	{
2240	/* Replace all RAM pages by ZERO pages. */
2241	while (iPage-- > 0)
2242	{
2243	PPGMPAGE pPage = &pRam->aPages[iPage];
2244	switch (PGM_PAGE_GET_TYPE(pPage))
2245	{
2246	case PGMPAGETYPE_RAM:
2247	/* Do not replace pages part of a 2 MB continuous range
2248	with zero pages, but zero them instead. */
2249	if ( PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE
2250	\|\| PGM_PAGE_GET_PDE_TYPE(pPage) == PGM_PAGE_PDE_TYPE_PDE_DISABLED)
2251	{
2252	void *pvPage;
2253	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2254	AssertLogRelRCReturn(rc, rc);
2255	RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2256	}
2257	else if (PGM_PAGE_IS_BALLOONED(pPage))
2258	{
2259	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2260	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2261	}
2262	else if (!PGM_PAGE_IS_ZERO(pPage))
2263	{
2264	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2265	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2266	AssertLogRelRCReturn(rc, rc);
2267	}
2268	break;
2269
2270	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2271	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2272	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2273	pRam, true /fDoAccounting/, false /fFlushIemTlbs/);
2274	break;
2275
2276	case PGMPAGETYPE_MMIO2:
2277	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2278	case PGMPAGETYPE_ROM:
2279	case PGMPAGETYPE_MMIO:
2280	break;
2281	default:
2282	AssertFailed();
2283	}
2284	} /* for each page */
2285	}
2286	else
2287	{
2288	/* Zero the memory. */
2289	while (iPage-- > 0)
2290	{
2291	PPGMPAGE pPage = &pRam->aPages[iPage];
2292	switch (PGM_PAGE_GET_TYPE(pPage))
2293	{
2294	case PGMPAGETYPE_RAM:
2295	switch (PGM_PAGE_GET_STATE(pPage))
2296	{
2297	case PGM_PAGE_STATE_ZERO:
2298	break;
2299
2300	case PGM_PAGE_STATE_BALLOONED:
2301	/* Turn into a zero page; the balloon status is lost when the VM reboots. */
2302	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
2303	break;
2304
2305	case PGM_PAGE_STATE_SHARED:
2306	case PGM_PAGE_STATE_WRITE_MONITORED:
2307	rc = pgmPhysPageMakeWritable(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
2308	AssertLogRelRCReturn(rc, rc);
2309	RT_FALL_THRU();
2310
2311	case PGM_PAGE_STATE_ALLOCATED:
2312	if (pVM->pgm.s.fZeroRamPagesOnReset)
2313	{
2314	void *pvPage;
2315	rc = pgmPhysPageMap(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pvPage);
2316	AssertLogRelRCReturn(rc, rc);
2317	RT_BZERO(pvPage, GUEST_PAGE_SIZE);
2318	}
2319	break;
2320	}
2321	break;
2322
2323	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2324	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO: /** @todo perhaps leave the special page alone? I don't think VT-x copes with this code. */
2325	pgmHandlerPhysicalResetAliasedPage(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT),
2326	pRam, true /fDoAccounting/, false /fFlushIemTlbs/);
2327	break;
2328
2329	case PGMPAGETYPE_MMIO2:
2330	case PGMPAGETYPE_ROM_SHADOW:
2331	case PGMPAGETYPE_ROM:
2332	case PGMPAGETYPE_MMIO:
2333	break;
2334	default:
2335	AssertFailed();
2336
2337	}
2338	} /* for each page */
2339	}
2340	}
2341
2342	/*
2343	* Finish off any pages pending freeing.
2344	*/
2345	if (cPendingPages)
2346	{
2347	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2348	AssertLogRelRCReturn(rc, rc);
2349	}
2350	GMMR3FreePagesCleanup(pReq);
2351
2352	/*
2353	* Flush the IEM TLB, just to be sure it really is done.
2354	*/
2355	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_ZERO_ALL);
2356
2357	return VINF_SUCCESS;
2358	}
2359
2360
2361	/**
2362	* Frees all RAM during VM termination
2363	*
2364	* ASSUMES that the caller owns the PGM lock.
2365	*
2366	* @returns VBox status code.
2367	* @param pVM The cross context VM structure.
2368	*/
2369	int pgmR3PhysRamTerm(PVM pVM)
2370	{
2371	PGM_LOCK_ASSERT_OWNER(pVM);
2372
2373	/* Reset the memory balloon. */
2374	int rc = GMMR3BalloonedPages(pVM, GMMBALLOONACTION_RESET, 0);
2375	AssertRC(rc);
2376
2377	#ifdef VBOX_WITH_PAGE_SHARING
2378	/*
2379	* Clear all registered shared modules.
2380	*/
2381	pgmR3PhysAssertSharedPageChecksums(pVM);
2382	rc = GMMR3ResetSharedModules(pVM);
2383	AssertRC(rc);
2384
2385	/*
2386	* Flush the handy pages updates to make sure no shared pages are hiding
2387	* in there. (Not unlikely if the VM shuts down, apparently.)
2388	*/
2389	# ifdef VBOX_WITH_PGM_NEM_MODE
2390	if (!pVM->pgm.s.fNemMode)
2391	# endif
2392	rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_FLUSH_HANDY_PAGES, 0, NULL);
2393	#endif
2394
2395	/*
2396	* We batch up pages that should be freed instead of calling GMM for
2397	* each and every one of them.
2398	*/
2399	uint32_t cPendingPages = 0;
2400	PGMMFREEPAGESREQ pReq;
2401	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
2402	AssertLogRelRCReturn(rc, rc);
2403
2404	/*
2405	* Walk the ram ranges.
2406	*/
2407	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
2408	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
2409	{
2410	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
2411	Assert(pRam \|\| idRamRange == 0);
2412	if (!pRam) continue;
2413	Assert(pRam->idRange == idRamRange);
2414
2415	uint32_t iPage = pRam->cb >> GUEST_PAGE_SHIFT;
2416	AssertMsg(((RTGCPHYS)iPage << GUEST_PAGE_SHIFT) == pRam->cb, ("%RGp %RGp\n", (RTGCPHYS)iPage << GUEST_PAGE_SHIFT, pRam->cb));
2417
2418	while (iPage-- > 0)
2419	{
2420	PPGMPAGE pPage = &pRam->aPages[iPage];
2421	switch (PGM_PAGE_GET_TYPE(pPage))
2422	{
2423	case PGMPAGETYPE_RAM:
2424	/* Free all shared pages. Private pages are automatically freed during GMM VM cleanup. */
2425	/** @todo change this to explicitly free private pages here. */
2426	if (PGM_PAGE_IS_SHARED(pPage))
2427	{
2428	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage,
2429	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), PGMPAGETYPE_RAM);
2430	AssertLogRelRCReturn(rc, rc);
2431	}
2432	break;
2433
2434	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
2435	case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
2436	case PGMPAGETYPE_MMIO2:
2437	case PGMPAGETYPE_ROM_SHADOW: /* handled by pgmR3PhysRomReset. */
2438	case PGMPAGETYPE_ROM:
2439	case PGMPAGETYPE_MMIO:
2440	break;
2441	default:
2442	AssertFailed();
2443	}
2444	} /* for each page */
2445	}
2446
2447	/*
2448	* Finish off any pages pending freeing.
2449	*/
2450	if (cPendingPages)
2451	{
2452	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
2453	AssertLogRelRCReturn(rc, rc);
2454	}
2455	GMMR3FreePagesCleanup(pReq);
2456	return VINF_SUCCESS;
2457	}
2458
2459
2460
2461	/*********************************************************************************************************************************
2462	* MMIO *
2463	*********************************************************************************************************************************/
2464
2465	/**
2466	* This is the interface IOM is using to register an MMIO region (unmapped).
2467	*
2468	*
2469	* @returns VBox status code.
2470	*
2471	* @param pVM The cross context VM structure.
2472	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2473	* @param cb The size of the MMIO region.
2474	* @param pszDesc The description of the MMIO region.
2475	* @param pidRamRange Where to return the RAM range ID for the MMIO region
2476	* on success.
2477	* @thread EMT(0)
2478	*/
2479	VMMR3_INT_DECL(int) PGMR3PhysMmioRegister(PVM pVM, PVMCPU pVCpu, RTGCPHYS cb, const char pszDesc, uint16_t pidRamRange)
2480	{
2481	/*
2482	* Assert assumptions.
2483	*/
2484	AssertPtrReturn(pidRamRange, VERR_INVALID_POINTER);
2485	*pidRamRange = UINT16_MAX;
2486	AssertReturn(pVCpu == VMMGetCpu(pVM) && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
2487	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
2488	/// @todo AssertReturn(!pVM->pgm.s.fRamRangesFrozen, VERR_WRONG_ORDER);
2489	AssertReturn(cb <= ((RTGCPHYS)PGM_MAX_PAGES_PER_RAM_RANGE << GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
2490	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2491	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
2492	AssertReturn(*pszDesc != '\0', VERR_INVALID_POINTER);
2493
2494	/*
2495	* Take the PGM lock and allocate an ad-hoc MMIO RAM range.
2496	*/
2497	int rc = PGM_LOCK(pVM);
2498	AssertRCReturn(rc, rc);
2499
2500	uint32_t const cPages = cb >> GUEST_PAGE_SHIFT;
2501	PPGMRAMRANGE pNew = NULL;
2502	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cPages, PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, &pNew);
2503	AssertLogRelMsg(RT_SUCCESS(rc), ("pgmR3PhysAllocateRamRange failed: cPages=%#RX32 (%s): %Rrc\n", cPages, pszDesc, rc));
2504	if (RT_SUCCESS(rc))
2505	{
2506	/* Initialize the range. */
2507	pNew->pszDesc = pszDesc;
2508	pNew->uNemRange = UINT32_MAX;
2509	pNew->pbR3 = NULL;
2510	pNew->paLSPages = NULL;
2511	Assert(pNew->fFlags == PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO && pNew->cb == cb);
2512
2513	uint32_t iPage = cPages;
2514	while (iPage-- > 0)
2515	PGM_PAGE_INIT_ZERO(&pNew->aPages[iPage], pVM, PGMPAGETYPE_MMIO);
2516	Assert(PGM_PAGE_GET_TYPE(&pNew->aPages[0]) == PGMPAGETYPE_MMIO);
2517
2518	/* update the page count stats. */
2519	pVM->pgm.s.cPureMmioPages += cPages;
2520	pVM->pgm.s.cAllPages += cPages;
2521
2522	/*
2523	* Set the return value, release lock and return to IOM.
2524	*/
2525	*pidRamRange = pNew->idRange;
2526	}
2527
2528	PGM_UNLOCK(pVM);
2529	return rc;
2530	}
2531
2532
2533	/**
2534	* Worker for PGMR3PhysMmioMap that's called owning the lock.
2535	*/
2536	static int pgmR3PhysMmioMapLocked(PVM pVM, PVMCPU pVCpu, RTGCPHYS const GCPhys, RTGCPHYS const cb, RTGCPHYS const GCPhysLast,
2537	PPGMRAMRANGE const pMmioRamRange, PGMPHYSHANDLERTYPE const hType, uint64_t const uUser)
2538	{
2539	/* Check that the range isn't mapped already. */
2540	AssertLogRelMsgReturn(pMmioRamRange->GCPhys == NIL_RTGCPHYS,
2541	("desired %RGp mapping for '%s' - already mapped at %RGp!\n",
2542	GCPhys, pMmioRamRange->pszDesc, pMmioRamRange->GCPhys),
2543	VERR_ALREADY_EXISTS);
2544
2545	/*
2546	* Now, check if this falls into a regular RAM range or if we should use
2547	* the ad-hoc one (idRamRange).
2548	*/
2549	int rc;
2550	uint32_t idxInsert = UINT32_MAX;
2551	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
2552	if (pOverlappingRange)
2553	{
2554	/* Simplification: all within the same range. */
2555	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
2556	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
2557	("%RGp-%RGp (MMIO/%s) falls partly outside %RGp-%RGp (%s)\n",
2558	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2559	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
2560	VERR_PGM_RAM_CONFLICT);
2561
2562	/* Check that is isn't an ad hoc range, but a real RAM range. */
2563	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
2564	("%RGp-%RGp (MMIO/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
2565	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2566	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
2567	VERR_PGM_RAM_CONFLICT);
2568
2569	/* Check that it's all RAM or MMIO pages. */
2570	PCPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
2571	uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2572	while (cLeft-- > 0)
2573	{
2574	AssertLogRelMsgReturn( PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM
2575	\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO, /** @todo MMIO type isn't right */
2576	("%RGp-%RGp (MMIO/%s): %RGp is not a RAM or MMIO page - type=%d desc=%s\n",
2577	GCPhys, GCPhysLast, pMmioRamRange->pszDesc, pOverlappingRange->GCPhys,
2578	PGM_PAGE_GET_TYPE(pPage), pOverlappingRange->pszDesc),
2579	VERR_PGM_RAM_CONFLICT);
2580	pPage++;
2581	}
2582
2583	/*
2584	* Make all the pages in the range MMIO/ZERO pages, freeing any
2585	* RAM pages currently mapped here. This might not be 100% correct
2586	* for PCI memory, but we're doing the same thing for MMIO2 pages.
2587	*/
2588	rc = pgmR3PhysFreePageRange(pVM, pOverlappingRange, GCPhys, GCPhysLast, NULL);
2589	AssertRCReturn(rc, rc);
2590
2591	/* Force a PGM pool flush as guest ram references have been changed. */
2592	/** @todo not entirely SMP safe; assuming for now the guest takes
2593	* care of this internally (not touch mapped mmio while changing the
2594	* mapping). */
2595	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2596	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2597	}
2598	else
2599	{
2600	/*
2601	* No RAM range, use the ad hoc one (idRamRange).
2602	*
2603	* Note that we don't have to tell REM about this range because
2604	* PGMHandlerPhysicalRegisterEx will do that for us.
2605	*/
2606	AssertLogRelReturn(idxInsert <= pVM->pgm.s.RamRangeUnion.cLookupEntries, VERR_INTERNAL_ERROR_4);
2607	Log(("PGMR3PhysMmioMap: Inserting ad hoc MMIO range #%x for %RGp-%RGp %s\n",
2608	pMmioRamRange->idRange, GCPhys, GCPhysLast, pMmioRamRange->pszDesc));
2609
2610	Assert(PGM_PAGE_GET_TYPE(&pMmioRamRange->aPages[0]) == PGMPAGETYPE_MMIO);
2611
2612	/* We ASSUME that all the pages in the ad-hoc range are in the proper
2613	state and all that and that we don't need to re-initialize them here. */
2614
2615	#ifdef VBOX_WITH_NATIVE_NEM
2616	/* Notify NEM. */
2617	if (VM_IS_NEM_ENABLED(pVM))
2618	{
2619	uint8_t u2State = 0; /* (must have valid state as there can't be anything to preserve) */
2620	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhys, cb, 0 /fFlags/, NULL, NULL, &u2State, &pMmioRamRange->uNemRange);
2621	AssertLogRelRCReturn(rc, rc);
2622
2623	uint32_t iPage = cb >> GUEST_PAGE_SHIFT;
2624	while (iPage-- > 0)
2625	PGM_PAGE_SET_NEM_STATE(&pMmioRamRange->aPages[iPage], u2State);
2626	}
2627	#endif
2628	/* Insert it into the lookup table (may in theory fail). */
2629	rc = pgmR3PhysRamRangeInsertLookup(pVM, pMmioRamRange, GCPhys, &idxInsert);
2630	}
2631	if (RT_SUCCESS(rc))
2632	{
2633	/*
2634	* Register the access handler.
2635	*/
2636	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, hType, uUser, pMmioRamRange->pszDesc);
2637	if (RT_SUCCESS(rc))
2638	{
2639	#ifdef VBOX_WITH_NATIVE_NEM
2640	/* Late NEM notification (currently not used by anyone). */
2641	if (VM_IS_NEM_ENABLED(pVM))
2642	{
2643	if (pOverlappingRange)
2644	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, cb, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
2645	pOverlappingRange->pbR3 + (uintptr_t)(GCPhys - pOverlappingRange->GCPhys),
2646	NULL /pvMmio2/, NULL /puNemRange/);
2647	else
2648	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, cb, 0 /fFlags/, NULL /pvRam/, NULL /pvMmio2/,
2649	&pMmioRamRange->uNemRange);
2650	AssertLogRelRC(rc);
2651	}
2652	if (RT_SUCCESS(rc))
2653	#endif
2654	{
2655	pgmPhysInvalidatePageMapTLB(pVM);
2656	return VINF_SUCCESS;
2657	}
2658
2659	/*
2660	* Failed, so revert it all as best as we can (the memory content in
2661	* the overlapping case is gone).
2662	*/
2663	PGMHandlerPhysicalDeregister(pVM, GCPhys);
2664	}
2665	}
2666
2667	if (!pOverlappingRange)
2668	{
2669	#ifdef VBOX_WITH_NATIVE_NEM
2670	/* Notify NEM about the sudden removal of the RAM range we just told it about. */
2671	NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, cb, 0 /fFlags/, NULL /pvRam/, NULL /pvMmio2/,
2672	NULL /pu2State/, &pMmioRamRange->uNemRange);
2673	#endif
2674
2675	/* Remove the ad hoc range from the lookup table. */
2676	idxInsert -= 1;
2677	pgmR3PhysRamRangeRemoveLookup(pVM, pMmioRamRange, &idxInsert);
2678	}
2679
2680	pgmPhysInvalidatePageMapTLB(pVM);
2681	return rc;
2682	}
2683
2684
2685	/**
2686	* This is the interface IOM is using to map an MMIO region.
2687	*
2688	* It will check for conflicts and ensure that a RAM range structure
2689	* is present before calling the PGMR3HandlerPhysicalRegister API to
2690	* register the callbacks.
2691	*
2692	* @returns VBox status code.
2693	*
2694	* @param pVM The cross context VM structure.
2695	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2696	* @param GCPhys The start of the MMIO region.
2697	* @param cb The size of the MMIO region.
2698	* @param idRamRange The RAM range ID for the MMIO region as returned by
2699	* PGMR3PhysMmioRegister().
2700	* @param hType The physical access handler type registration.
2701	* @param uUser The user argument.
2702	* @thread EMT(pVCpu)
2703	*/
2704	VMMR3_INT_DECL(int) PGMR3PhysMmioMap(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, uint16_t idRamRange,
2705	PGMPHYSHANDLERTYPE hType, uint64_t uUser)
2706	{
2707	/*
2708	* Assert on some assumption.
2709	*/
2710	VMCPU_ASSERT_EMT(pVCpu);
2711	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2712	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2713	RTGCPHYS const GCPhysLast = GCPhys + cb - 1U;
2714	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2715	#ifdef VBOX_STRICT
2716	PCPGMPHYSHANDLERTYPEINT pType = pgmHandlerPhysicalTypeHandleToPtr(pVM, hType);
2717	Assert(pType);
2718	Assert(pType->enmKind == PGMPHYSHANDLERKIND_MMIO);
2719	#endif
2720	AssertReturn(idRamRange <= pVM->pgm.s.idRamRangeMax && idRamRange > 0, VERR_INVALID_HANDLE);
2721	PPGMRAMRANGE const pMmioRamRange = pVM->pgm.s.apRamRanges[idRamRange];
2722	AssertReturn(pMmioRamRange, VERR_INVALID_HANDLE);
2723	AssertReturn(pMmioRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, VERR_INVALID_HANDLE);
2724	AssertReturn(pMmioRamRange->cb == cb, VERR_OUT_OF_RANGE);
2725
2726	/*
2727	* Take the PGM lock and do the work.
2728	*/
2729	int rc = PGM_LOCK(pVM);
2730	AssertRCReturn(rc, rc);
2731
2732	rc = pgmR3PhysMmioMapLocked(pVM, pVCpu, GCPhys, cb, GCPhysLast, pMmioRamRange, hType, uUser);
2733	#ifdef VBOX_STRICT
2734	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2735	#endif
2736
2737	PGM_UNLOCK(pVM);
2738	return rc;
2739	}
2740
2741
2742	/**
2743	* Worker for PGMR3PhysMmioUnmap that's called with the PGM lock held.
2744	*/
2745	static int pgmR3PhysMmioUnmapLocked(PVM pVM, PVMCPU pVCpu, RTGCPHYS const GCPhys, RTGCPHYS const cb,
2746	RTGCPHYS const GCPhysLast, PPGMRAMRANGE const pMmioRamRange)
2747	{
2748	/*
2749	* Lookup the RAM range containing the region to make sure it is actually mapped.
2750	*/
2751	uint32_t idxLookup = pgmR3PhysRamRangeFindOverlappingIndex(pVM, GCPhys, GCPhysLast);
2752	AssertLogRelMsgReturn(idxLookup < pVM->pgm.s.RamRangeUnion.cLookupEntries,
2753	("MMIO range not found at %RGp LB %RGp! (%s)\n", GCPhys, cb, pMmioRamRange->pszDesc),
2754	VERR_NOT_FOUND);
2755
2756	uint32_t const idLookupRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
2757	AssertLogRelReturn(idLookupRange != 0 && idLookupRange <= pVM->pgm.s.idRamRangeMax, VERR_INTERNAL_ERROR_5);
2758	PPGMRAMRANGE const pLookupRange = pVM->pgm.s.apRamRanges[idLookupRange];
2759	AssertLogRelReturn(pLookupRange, VERR_INTERNAL_ERROR_4);
2760
2761	AssertLogRelMsgReturn(pLookupRange == pMmioRamRange \|\| !PGM_RAM_RANGE_IS_AD_HOC(pLookupRange),
2762	("MMIO unmap mixup at %RGp LB %RGp (%s) vs %RGp LB %RGp (%s)\n",
2763	GCPhys, cb, pMmioRamRange->pszDesc, pLookupRange->GCPhys, pLookupRange->cb, pLookupRange->pszDesc),
2764	VERR_NOT_FOUND);
2765
2766	/*
2767	* Deregister the handler. This should reset any aliases, so an ad hoc
2768	* range will only contain MMIO type pages afterwards.
2769	*/
2770	int rc = PGMHandlerPhysicalDeregister(pVM, GCPhys);
2771	if (RT_SUCCESS(rc))
2772	{
2773	if (pLookupRange != pMmioRamRange)
2774	{
2775	/*
2776	* Turn the pages back into RAM pages.
2777	*/
2778	Log(("pgmR3PhysMmioUnmapLocked: Reverting MMIO range %RGp-%RGp (%s) in %RGp-%RGp (%s) to RAM.\n",
2779	GCPhys, GCPhysLast, pMmioRamRange->pszDesc,
2780	pLookupRange->GCPhys, pLookupRange->GCPhysLast, pLookupRange->pszDesc));
2781
2782	RTGCPHYS const offRange = GCPhys - pLookupRange->GCPhys;
2783	uint32_t iPage = offRange >> GUEST_PAGE_SHIFT;
2784	uint32_t cLeft = cb >> GUEST_PAGE_SHIFT;
2785	while (cLeft--)
2786	{
2787	PPGMPAGE pPage = &pLookupRange->aPages[iPage];
2788	AssertMsg( (PGM_PAGE_IS_MMIO(pPage) && PGM_PAGE_IS_ZERO(pPage))
2789	//\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_MMIO2_ALIAS_MMIO
2790	//\|\| PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_SPECIAL_ALIAS_MMIO
2791	, ("%RGp %R[pgmpage]\n", pLookupRange->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), pPage));
2792	/** @todo this isn't entirely correct, is it now... aliases must be converted
2793	* to zero pages as they won't be. however, shouldn't
2794	* PGMHandlerPhysicalDeregister deal with this already? */
2795	if (PGM_PAGE_IS_MMIO_OR_ALIAS(pPage))
2796	PGM_PAGE_SET_TYPE(pVM, pPage, PGMPAGETYPE_RAM);
2797	iPage++;
2798	}
2799
2800	#ifdef VBOX_WITH_NATIVE_NEM
2801	/* Notify REM (failure will probably leave things in a non-working state). */
2802	if (VM_IS_NEM_ENABLED(pVM))
2803	{
2804	uint8_t u2State = UINT8_MAX;
2805	rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE,
2806	pLookupRange->pbR3 ? pLookupRange->pbR3 + GCPhys - pLookupRange->GCPhys : NULL,
2807	NULL, &u2State, &pLookupRange->uNemRange);
2808	AssertLogRelRC(rc);
2809	/** @todo status code propagation here... This is likely fatal, right? */
2810	if (u2State != UINT8_MAX)
2811	pgmPhysSetNemStateForPages(&pLookupRange->aPages[(GCPhys - pLookupRange->GCPhys) >> GUEST_PAGE_SHIFT],
2812	cb >> GUEST_PAGE_SHIFT, u2State);
2813	}
2814	#endif
2815	}
2816	else
2817	{
2818	/*
2819	* Unlink the ad hoc range.
2820	*/
2821	#ifdef VBOX_STRICT
2822	uint32_t iPage = cb >> GUEST_PAGE_SHIFT;
2823	while (iPage-- > 0)
2824	{
2825	PPGMPAGE const pPage = &pMmioRamRange->aPages[iPage];
2826	Assert(PGM_PAGE_IS_MMIO(pPage));
2827	}
2828	#endif
2829
2830	Log(("pgmR3PhysMmioUnmapLocked: Unmapping ad hoc MMIO range for %RGp-%RGp %s\n",
2831	GCPhys, GCPhysLast, pMmioRamRange->pszDesc));
2832
2833	#ifdef VBOX_WITH_NATIVE_NEM
2834	if (VM_IS_NEM_ENABLED(pVM)) /* Notify REM before we unlink the range. */
2835	{
2836	rc = NEMR3NotifyPhysMmioExUnmap(pVM, GCPhys, GCPhysLast - GCPhys + 1, 0 /fFlags/,
2837	NULL, NULL, NULL, &pMmioRamRange->uNemRange);
2838	AssertLogRelRCReturn(rc, rc); /* we're up the creek if this hits. */
2839	}
2840	#endif
2841
2842	pgmR3PhysRamRangeRemoveLookup(pVM, pMmioRamRange, &idxLookup);
2843	}
2844	}
2845
2846	/* Force a PGM pool flush as guest ram references have been changed. */
2847	/** @todo Not entirely SMP safe; assuming for now the guest takes care of
2848	* this internally (not touch mapped mmio while changing the mapping). */
2849	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
2850	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
2851
2852	pgmPhysInvalidatePageMapTLB(pVM);
2853	pgmPhysInvalidRamRangeTlbs(pVM);
2854
2855	return rc;
2856	}
2857
2858
2859	/**
2860	* This is the interface IOM is using to register an MMIO region.
2861	*
2862	* It will take care of calling PGMHandlerPhysicalDeregister and clean up
2863	* any ad hoc PGMRAMRANGE left behind.
2864	*
2865	* @returns VBox status code.
2866	* @param pVM The cross context VM structure.
2867	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
2868	* @param GCPhys The start of the MMIO region.
2869	* @param cb The size of the MMIO region.
2870	* @param idRamRange The RAM range ID for the MMIO region as returned by
2871	* PGMR3PhysMmioRegister().
2872	* @thread EMT(pVCpu)
2873	*/
2874	VMMR3_INT_DECL(int) PGMR3PhysMmioUnmap(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys, RTGCPHYS cb, uint16_t idRamRange)
2875	{
2876	/*
2877	* Input validation.
2878	*/
2879	VMCPU_ASSERT_EMT(pVCpu);
2880	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2881	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
2882	RTGCPHYS const GCPhysLast = GCPhys + cb - 1U;
2883	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
2884	AssertReturn(idRamRange <= pVM->pgm.s.idRamRangeMax && idRamRange > 0, VERR_INVALID_HANDLE);
2885	PPGMRAMRANGE const pMmioRamRange = pVM->pgm.s.apRamRanges[idRamRange];
2886	AssertReturn(pMmioRamRange, VERR_INVALID_HANDLE);
2887	AssertReturn(pMmioRamRange->fFlags & PGM_RAM_RANGE_FLAGS_AD_HOC_MMIO, VERR_INVALID_HANDLE);
2888	AssertReturn(pMmioRamRange->cb == cb, VERR_OUT_OF_RANGE);
2889
2890	/*
2891	* Take the PGM lock and do what's asked.
2892	*/
2893	int rc = PGM_LOCK(pVM);
2894	AssertRCReturn(rc, rc);
2895
2896	rc = pgmR3PhysMmioUnmapLocked(pVM, pVCpu, GCPhys, cb, GCPhysLast, pMmioRamRange);
2897	#ifdef VBOX_STRICT
2898	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
2899	#endif
2900
2901	PGM_UNLOCK(pVM);
2902	return rc;
2903	}
2904
2905
2906
2907	/*********************************************************************************************************************************
2908	* MMIO2 *
2909	*********************************************************************************************************************************/
2910
2911	/**
2912	* Validates the claim to an MMIO2 range and returns the pointer to it.
2913	*
2914	* @returns The MMIO2 entry index on success, negative error status on failure.
2915	* @param pVM The cross context VM structure.
2916	* @param pDevIns The device instance owning the region.
2917	* @param hMmio2 Handle to look up.
2918	* @param pcChunks Where to return the number of chunks associated with
2919	* this handle.
2920	*/
2921	static int32_t pgmR3PhysMmio2ResolveHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t *pcChunks)
2922	{
2923	*pcChunks = 0;
2924	uint32_t const idxFirst = hMmio2 - 1U;
2925	uint32_t const cMmio2Ranges = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
2926	AssertReturn(idxFirst < cMmio2Ranges, VERR_INVALID_HANDLE);
2927
2928	PPGMREGMMIO2RANGE const pFirst = &pVM->pgm.s.aMmio2Ranges[idxFirst];
2929	AssertReturn(pFirst->idMmio2 == hMmio2, VERR_INVALID_HANDLE);
2930	AssertReturn((pFirst->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK), VERR_INVALID_HANDLE);
2931	AssertReturn(pFirst->pDevInsR3 == pDevIns && RT_VALID_PTR(pDevIns), VERR_NOT_OWNER);
2932
2933	/* Figure out how many chunks this handle spans. */
2934	if (pFirst->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2935	*pcChunks = 1;
2936	else
2937	{
2938	uint32_t cChunks = 1;
2939	for (uint32_t idx = idxFirst + 1;; idx++)
2940	{
2941	cChunks++;
2942	AssertReturn(idx < cMmio2Ranges, VERR_INTERNAL_ERROR_2);
2943	PPGMREGMMIO2RANGE const pCur = &pVM->pgm.s.aMmio2Ranges[idx];
2944	AssertLogRelMsgReturn( pCur->pDevInsR3 == pDevIns
2945	&& pCur->idMmio2 == idx + 1
2946	&& pCur->iSubDev == pFirst->iSubDev
2947	&& pCur->iRegion == pFirst->iRegion
2948	&& !(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK),
2949	("cur: %p/%#x/%#x/%#x/%#x/%s; first: %p/%#x/%#x/%#x/%#x/%s\n",
2950	pCur->pDevInsR3, pCur->idMmio2, pCur->iSubDev, pCur->iRegion, pCur->fFlags,
2951	pVM->pgm.s.apMmio2RamRanges[idx]->pszDesc,
2952	pDevIns, idx + 1, pFirst->iSubDev, pFirst->iRegion, pFirst->fFlags,
2953	pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc),
2954	VERR_INTERNAL_ERROR_3);
2955	if (pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
2956	break;
2957	}
2958	*pcChunks = cChunks;
2959	}
2960
2961	return (int32_t)idxFirst;
2962	}
2963
2964
2965	/**
2966	* Check if a device has already registered a MMIO2 region.
2967	*
2968	* @returns NULL if not registered, otherwise pointer to the MMIO2.
2969	* @param pVM The cross context VM structure.
2970	* @param pDevIns The device instance owning the region.
2971	* @param iSubDev The sub-device number.
2972	* @param iRegion The region.
2973	*/
2974	DECLINLINE(PPGMREGMMIO2RANGE) pgmR3PhysMmio2Find(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion)
2975	{
2976	/*
2977	* Search the array. There shouldn't be many entries.
2978	*/
2979	uint32_t idx = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
2980	while (idx-- > 0)
2981	if (RT_LIKELY( pVM->pgm.s.aMmio2Ranges[idx].pDevInsR3 != pDevIns
2982	\|\| pVM->pgm.s.aMmio2Ranges[idx].iRegion != iRegion
2983	\|\| pVM->pgm.s.aMmio2Ranges[idx].iSubDev != iSubDev))
2984	{ /* likely */ }
2985	else
2986	return &pVM->pgm.s.aMmio2Ranges[idx];
2987	return NULL;
2988	}
2989
2990	/**
2991	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Map.
2992	*/
2993	static int pgmR3PhysMmio2EnableDirtyPageTracing(PVM pVM, uint32_t idx, uint32_t cChunks)
2994	{
2995	int rc = VINF_SUCCESS;
2996	while (cChunks-- > 0)
2997	{
2998	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
2999	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3000
3001	Assert(!(pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING));
3002	int rc2 = pgmHandlerPhysicalExRegister(pVM, pMmio2->pPhysHandlerR3, pRamRange->GCPhys, pRamRange->GCPhysLast);
3003	if (RT_SUCCESS(rc2))
3004	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_IS_TRACKING;
3005	else
3006	AssertLogRelMsgFailedStmt(("%#RGp-%#RGp %s failed -> %Rrc\n",
3007	pRamRange->GCPhys, pRamRange->GCPhysLast, pRamRange->pszDesc, rc2),
3008	rc = RT_SUCCESS(rc) ? rc2 : rc);
3009
3010	idx++;
3011	}
3012	return rc;
3013	}
3014
3015
3016	/**
3017	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking and PGMR3PhysMmio2Unmap.
3018	*/
3019	static int pgmR3PhysMmio2DisableDirtyPageTracing(PVM pVM, uint32_t idx, uint32_t cChunks)
3020	{
3021	int rc = VINF_SUCCESS;
3022	while (cChunks-- > 0)
3023	{
3024	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3025	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3026	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_TRACKING)
3027	{
3028	int rc2 = pgmHandlerPhysicalExDeregister(pVM, pMmio2->pPhysHandlerR3);
3029	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3030	("%#RGp-%#RGp %s failed -> %Rrc\n",
3031	pRamRange->GCPhys, pRamRange->GCPhysLast, pRamRange->pszDesc, rc2),
3032	rc = RT_SUCCESS(rc) ? rc2 : rc);
3033	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_TRACKING;
3034	}
3035	idx++;
3036	}
3037	return rc;
3038	}
3039
3040	#if 0 // temp
3041
3042	/**
3043	* Common worker PGMR3PhysMmio2PreRegister & PGMR3PhysMMIO2Register that links a
3044	* complete registration entry into the lists and lookup tables.
3045	*
3046	* @param pVM The cross context VM structure.
3047	* @param pNew The new MMIO / MMIO2 registration to link.
3048	*/
3049	static void pgmR3PhysMmio2Link(PVM pVM, PPGMREGMMIO2RANGE pNew)
3050	{
3051	Assert(pNew->idMmio2 != UINT8_MAX);
3052
3053	/*
3054	* Link it into the list (order doesn't matter, so insert it at the head).
3055	*
3056	* Note! The range we're linking may consist of multiple chunks, so we
3057	* have to find the last one.
3058	*/
3059	PPGMREGMMIO2RANGE pLast = pNew;
3060	for (pLast = pNew; ; pLast = pLast->pNextR3)
3061	{
3062	if (pLast->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3063	break;
3064	Assert(pLast->pNextR3);
3065	Assert(pLast->pNextR3->pDevInsR3 == pNew->pDevInsR3);
3066	Assert(pLast->pNextR3->iSubDev == pNew->iSubDev);
3067	Assert(pLast->pNextR3->iRegion == pNew->iRegion);
3068	Assert(pLast->pNextR3->idMmio2 == pLast->idMmio2 + 1);
3069	}
3070
3071	PGM_LOCK_VOID(pVM);
3072
3073	/* Link in the chain of ranges at the head of the list. */
3074	pLast->pNextR3 = pVM->pgm.s.pRegMmioRangesR3;
3075	pVM->pgm.s.pRegMmioRangesR3 = pNew;
3076
3077	/* Insert the MMIO2 range/page IDs. */
3078	uint8_t idMmio2 = pNew->idMmio2;
3079	for (;;)
3080	{
3081	Assert(pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] == NULL);
3082	Assert(pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] == NIL_RTR0PTR);
3083	pVM->pgm.s.apMmio2RangesR3[idMmio2 - 1] = pNew;
3084	pVM->pgm.s.apMmio2RangesR0[idMmio2 - 1] = pNew->RamRange.pSelfR0 - RT_UOFFSETOF(PGMREGMMIO2RANGE, RamRange);
3085	if (pNew->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK)
3086	break;
3087	pNew = pNew->pNextR3;
3088	idMmio2++;
3089	}
3090
3091	pgmPhysInvalidatePageMapTLB(pVM);
3092	PGM_UNLOCK(pVM);
3093	}
3094	#endif
3095
3096
3097	/**
3098	* Allocate and register an MMIO2 region.
3099	*
3100	* As mentioned elsewhere, MMIO2 is just RAM spelled differently. It's RAM
3101	* associated with a device. It is also non-shared memory with a permanent
3102	* ring-3 mapping and page backing (presently).
3103	*
3104	* A MMIO2 range may overlap with base memory if a lot of RAM is configured for
3105	* the VM, in which case we'll drop the base memory pages. Presently we will
3106	* make no attempt to preserve anything that happens to be present in the base
3107	* memory that is replaced, this is of course incorrect but it's too much
3108	* effort.
3109	*
3110	* @returns VBox status code.
3111	* @retval VINF_SUCCESS on success, *ppv pointing to the R3 mapping of the
3112	* memory.
3113	* @retval VERR_ALREADY_EXISTS if the region already exists.
3114	*
3115	* @param pVM The cross context VM structure.
3116	* @param pDevIns The device instance owning the region.
3117	* @param iSubDev The sub-device number.
3118	* @param iRegion The region number. If the MMIO2 memory is a PCI
3119	* I/O region this number has to be the number of that
3120	* region. Otherwise it can be any number save
3121	* UINT8_MAX.
3122	* @param cb The size of the region. Must be page aligned.
3123	* @param fFlags Reserved for future use, must be zero.
3124	* @param pszDesc The description.
3125	* @param ppv Where to store the pointer to the ring-3 mapping of
3126	* the memory.
3127	* @param phRegion Where to return the MMIO2 region handle. Optional.
3128	* @thread EMT(0)
3129	*
3130	* @note Only callable at VM creation time and during VM state loading.
3131	* The latter is for PCNet saved state compatibility with pre 4.3.6
3132	* state.
3133	*/
3134	VMMR3_INT_DECL(int) PGMR3PhysMmio2Register(PVM pVM, PPDMDEVINS pDevIns, uint32_t iSubDev, uint32_t iRegion, RTGCPHYS cb,
3135	uint32_t fFlags, const char pszDesc, void ppv, PGMMMIO2HANDLE phRegion)
3136	{
3137	/*
3138	* Validate input.
3139	*/
3140	AssertPtrReturn(ppv, VERR_INVALID_POINTER);
3141	*ppv = NULL;
3142	if (phRegion)
3143	{
3144	AssertPtrReturn(phRegion, VERR_INVALID_POINTER);
3145	*phRegion = NIL_PGMMMIO2HANDLE;
3146	}
3147	PVMCPU const pVCpu = VMMGetCpu(pVM);
3148	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
3149	VMSTATE const enmVMState = VMR3GetState(pVM);
3150	AssertMsgReturn(enmVMState == VMSTATE_CREATING \|\| enmVMState == VMSTATE_LOADING,
3151	("state %s, expected CREATING or LOADING\n", VMGetStateName(enmVMState)),
3152	VERR_VM_INVALID_VM_STATE);
3153
3154	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3155	AssertReturn(iSubDev <= UINT8_MAX, VERR_INVALID_PARAMETER);
3156	AssertReturn(iRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
3157
3158	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
3159	AssertReturn(*pszDesc, VERR_INVALID_PARAMETER);
3160
3161	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3162	AssertReturn(cb, VERR_INVALID_PARAMETER);
3163	AssertReturn(!(fFlags & ~PGMPHYS_MMIO2_FLAGS_VALID_MASK), VERR_INVALID_FLAGS);
3164
3165	const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
3166	AssertLogRelReturn(((RTGCPHYS)cGuestPages << GUEST_PAGE_SHIFT) == cb, VERR_INVALID_PARAMETER);
3167	AssertLogRelReturn(cGuestPages <= PGM_MAX_PAGES_PER_MMIO2_REGION, VERR_OUT_OF_RANGE);
3168	AssertLogRelReturn(cGuestPages <= (MM_MMIO_64_MAX >> GUEST_PAGE_SHIFT), VERR_OUT_OF_RANGE);
3169
3170	AssertReturn(pgmR3PhysMmio2Find(pVM, pDevIns, iSubDev, iRegion) == NULL, VERR_ALREADY_EXISTS);
3171
3172	/*
3173	* For the 2nd+ instance, mangle the description string so it's unique.
3174	*/
3175	if (pDevIns->iInstance > 0) /** @todo Move to PDMDevHlp.cpp and use a real string cache. */
3176	{
3177	pszDesc = MMR3HeapAPrintf(pVM, MM_TAG_PGM_PHYS, "%s [%u]", pszDesc, pDevIns->iInstance);
3178	if (!pszDesc)
3179	return VERR_NO_MEMORY;
3180	}
3181
3182	/*
3183	* Check that we've got sufficient MMIO2 ID space for this request (the
3184	* allocation will be done later once we've got the backing memory secured,
3185	* but given the EMT0 restriction, that's not going to be a problem).
3186	*
3187	* The zero ID is not used as it could be confused with NIL_GMM_PAGEID, so
3188	* the IDs goes from 1 thru PGM_MAX_MMIO2_RANGES.
3189	*/
3190	unsigned const cChunks = pgmPhysMmio2CalcChunkCount(cb, NULL);
3191
3192	int rc = PGM_LOCK(pVM);
3193	AssertRCReturn(rc, rc);
3194
3195	AssertCompile(PGM_MAX_MMIO2_RANGES < 255);
3196	uint8_t const idMmio2 = pVM->pgm.s.cMmio2Ranges + 1;
3197	AssertLogRelReturnStmt(idMmio2 + cChunks <= PGM_MAX_MMIO2_RANGES, PGM_UNLOCK(pVM), VERR_PGM_TOO_MANY_MMIO2_RANGES);
3198
3199	/*
3200	* Try reserve and allocate the backing memory first as this is what is
3201	* most likely to fail.
3202	*/
3203	rc = MMR3AdjustFixedReservation(pVM, cGuestPages, pszDesc);
3204	if (RT_SUCCESS(rc))
3205	{
3206	/*
3207	* If we're in driverless we'll be doing the work here, otherwise we
3208	* must call ring-0 to do the job as we'll need physical addresses
3209	* and maybe a ring-0 mapping address for it all.
3210	*/
3211	if (SUPR3IsDriverless())
3212	rc = pgmPhysMmio2RegisterWorker(pVM, cGuestPages, idMmio2, cChunks, pDevIns, iSubDev, iRegion, fFlags);
3213	else
3214	{
3215	PGMPHYSMMIO2REGISTERREQ Mmio2RegReq;
3216	Mmio2RegReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3217	Mmio2RegReq.Hdr.cbReq = sizeof(Mmio2RegReq);
3218	Mmio2RegReq.cbGuestPage = GUEST_PAGE_SIZE;
3219	Mmio2RegReq.cGuestPages = cGuestPages;
3220	Mmio2RegReq.idMmio2 = idMmio2;
3221	Mmio2RegReq.cChunks = cChunks;
3222	Mmio2RegReq.iSubDev = (uint8_t)iSubDev;
3223	Mmio2RegReq.iRegion = (uint8_t)iRegion;
3224	Mmio2RegReq.fFlags = fFlags;
3225	Mmio2RegReq.pDevIns = pDevIns;
3226	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_MMIO2_REGISTER, 0 /u64Arg/, &Mmio2RegReq.Hdr);
3227	}
3228	if (RT_SUCCESS(rc))
3229	{
3230	Assert(idMmio2 + cChunks - 1 == pVM->pgm.s.cMmio2Ranges);
3231
3232	/*
3233	* There are two things left to do:
3234	* 1. Add the description to the associated RAM ranges.
3235	* 2. Pre-allocate access handlers for dirty bit tracking if necessary.
3236	*/
3237	bool const fNeedHandler = (fFlags & PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES)
3238	#ifdef VBOX_WITH_PGM_NEM_MODE
3239	&& (!VM_IS_NEM_ENABLED(pVM) \|\| !NEMR3IsMmio2DirtyPageTrackingSupported(pVM))
3240	#endif
3241	;
3242	for (uint32_t idxChunk = 0; idxChunk < cChunks; idxChunk++)
3243	{
3244	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idxChunk + idMmio2 - 1];
3245	Assert(pMmio2->idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
3246	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apRamRanges[pMmio2->idRamRange];
3247	Assert(pRamRange->pbR3 == pMmio2->pbR3);
3248	Assert(pRamRange->cb == pMmio2->cbReal);
3249
3250	pRamRange->pszDesc = pszDesc; /** @todo mangle this if we got more than one chunk */
3251	if (fNeedHandler)
3252	{
3253	rc = pgmHandlerPhysicalExCreate(pVM, pVM->pgm.s.hMmio2DirtyPhysHandlerType, pMmio2->idMmio2,
3254	pszDesc, &pMmio2->pPhysHandlerR3);
3255	AssertLogRelMsgReturnStmt(RT_SUCCESS(rc),
3256	("idMmio2=%#x idxChunk=%#x rc=%Rc\n", idMmio2, idxChunk, rc),
3257	PGM_UNLOCK(pVM),
3258	rc); /* PGMR3Term will take care of it all. */
3259	}
3260	}
3261
3262	/*
3263	* Done!
3264	*/
3265	if (phRegion)
3266	*phRegion = idMmio2;
3267	*ppv = pVM->pgm.s.aMmio2Ranges[idMmio2 - 1].pbR3;
3268
3269	PGM_UNLOCK(pVM);
3270	return VINF_SUCCESS;
3271	}
3272
3273	MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pszDesc);
3274	}
3275	if (pDevIns->iInstance > 0)
3276	MMR3HeapFree((void *)pszDesc);
3277	return rc;
3278	}
3279
3280	/**
3281	* Deregisters and frees an MMIO2 region.
3282	*
3283	* Any physical access handlers registered for the region must be deregistered
3284	* before calling this function.
3285	*
3286	* @returns VBox status code.
3287	* @param pVM The cross context VM structure.
3288	* @param pDevIns The device instance owning the region.
3289	* @param hMmio2 The MMIO2 handle to deregister, or NIL if all
3290	* regions for the given device is to be deregistered.
3291	* @thread EMT(0)
3292	*
3293	* @note Only callable during VM state loading. This is to jettison an unused
3294	* MMIO2 section present in PCNet saved state prior to VBox v4.3.6.
3295	*/
3296	VMMR3_INT_DECL(int) PGMR3PhysMmio2Deregister(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
3297	{
3298	/*
3299	* Validate input.
3300	*/
3301	PVMCPU const pVCpu = VMMGetCpu(pVM);
3302	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
3303	VMSTATE const enmVMState = VMR3GetState(pVM);
3304	AssertMsgReturn(enmVMState == VMSTATE_LOADING,
3305	("state %s, expected LOADING\n", VMGetStateName(enmVMState)),
3306	VERR_VM_INVALID_VM_STATE);
3307
3308	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3309
3310	/*
3311	* Take the PGM lock and scan for registrations matching the requirements.
3312	* We do this backwards to more easily reduce the cMmio2Ranges count when
3313	* stuff is removed.
3314	*/
3315	PGM_LOCK_VOID(pVM);
3316
3317	int rc = VINF_SUCCESS;
3318	unsigned cFound = 0;
3319	uint32_t const cMmio2Ranges = RT_MIN(pVM->pgm.s.cMmio2Ranges, RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges));
3320	uint32_t idx = cMmio2Ranges;
3321	while (idx-- > 0)
3322	{
3323	PPGMREGMMIO2RANGE pCur = &pVM->pgm.s.aMmio2Ranges[idx];
3324	if ( pCur->pDevInsR3 == pDevIns
3325	&& ( hMmio2 == NIL_PGMMMIO2HANDLE
3326	\|\| pCur->idMmio2 == hMmio2))
3327	{
3328	cFound++;
3329
3330	/*
3331	* Wind back the first chunk for this registration.
3332	*/
3333	AssertLogRelMsgReturnStmt(pCur->fFlags & PGMREGMMIO2RANGE_F_LAST_CHUNK, ("idx=%u fFlags=%#x\n", idx, pCur->fFlags),
3334	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3335	uint32_t cGuestPages = pCur->cbReal >> GUEST_PAGE_SHIFT;
3336	uint32_t cChunks = 1;
3337	while ( idx > 0
3338	&& !(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK))
3339	{
3340	AssertLogRelMsgReturnStmt( pCur[-1].pDevInsR3 == pDevIns
3341	&& pCur[-1].iRegion == pCur->iRegion
3342	&& pCur[-1].iSubDev == pCur->iSubDev,
3343	("[%u]: %p/%#x/%#x/fl=%#x; [%u]: %p/%#x/%#x/fl=%#x; cChunks=%#x\n",
3344	idx - 1, pCur[-1].pDevInsR3, pCur[-1].iRegion, pCur[-1].iSubDev, pCur[-1].fFlags,
3345	idx, pCur->pDevInsR3, pCur->iRegion, pCur->iSubDev, pCur->fFlags, cChunks),
3346	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3347	cChunks++;
3348	pCur--;
3349	idx--;
3350	cGuestPages += pCur->cbReal >> GUEST_PAGE_SHIFT;
3351	}
3352	AssertLogRelMsgReturnStmt(pCur->fFlags & PGMREGMMIO2RANGE_F_FIRST_CHUNK,
3353	("idx=%u fFlags=%#x cChunks=%#x\n", idx, pCur->fFlags, cChunks),
3354	PGM_UNLOCK(pVM), VERR_INTERNAL_ERROR_3);
3355
3356	/*
3357	* Unmap it if it's mapped.
3358	*/
3359	if (pCur->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
3360	{
3361	int rc2 = PGMR3PhysMmio2Unmap(pVM, pCur->pDevInsR3, idx + 1, pCur->GCPhys);
3362	AssertRC(rc2);
3363	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
3364	rc = rc2;
3365	}
3366
3367	/*
3368	* Destroy access handlers.
3369	*/
3370	for (uint32_t iChunk = 0; iChunk < cChunks; iChunk++)
3371	if (pCur[iChunk].pPhysHandlerR3)
3372	{
3373	pgmHandlerPhysicalExDestroy(pVM, pCur[iChunk].pPhysHandlerR3);
3374	pCur[iChunk].pPhysHandlerR3 = NULL;
3375	}
3376
3377	/*
3378	* Call kernel mode / worker to do the actual deregistration.
3379	*/
3380	const char * const pszDesc = pVM->pgm.s.apMmio2RamRanges[idx] ? pVM->pgm.s.apMmio2RamRanges[idx]->pszDesc : NULL;
3381	int rc2;
3382	if (SUPR3IsDriverless())
3383	{
3384	Assert(PGM_IS_IN_NEM_MODE(pVM));
3385	rc2 = pgmPhysMmio2DeregisterWorker(pVM, idx, cChunks, pDevIns);
3386	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3387	("pgmPhysMmio2DeregisterWorker: rc=%Rrc idx=%#x cChunks=%#x %s\n",
3388	rc2, idx, cChunks, pszDesc),
3389	rc = RT_SUCCESS(rc) ? rc2 : rc);
3390	}
3391	else
3392	{
3393	PGMPHYSMMIO2DEREGISTERREQ Mmio2DeregReq;
3394	Mmio2DeregReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
3395	Mmio2DeregReq.Hdr.cbReq = sizeof(Mmio2DeregReq);
3396	Mmio2DeregReq.idMmio2 = idx + 1;
3397	Mmio2DeregReq.cChunks = cChunks;
3398	Mmio2DeregReq.pDevIns = pDevIns;
3399	rc2 = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_MMIO2_DEREGISTER, 0 /u64Arg/, &Mmio2DeregReq.Hdr);
3400	AssertLogRelMsgStmt(RT_SUCCESS(rc2),
3401	("VMMR0_DO_PGM_PHYS_MMIO2_DEREGISTER: rc=%Rrc idx=%#x cChunks=%#x %s\n",
3402	rc2, idx, cChunks, pszDesc),
3403	rc = RT_SUCCESS(rc) ? rc2 : rc);
3404	}
3405	if (RT_FAILURE(rc2))
3406	{
3407	LogRel(("PGMR3PhysMmio2Deregister: Deregistration failed: %Rrc; cChunks=%u %s\n", rc, cChunks, pszDesc));
3408	if (RT_SUCCESS(rc))
3409	rc = rc2;
3410	}
3411
3412	/*
3413	* Adjust the memory reservation.
3414	*/
3415	if (!PGM_IS_IN_NEM_MODE(pVM) && RT_SUCCESS(rc2))
3416	{
3417	rc2 = MMR3AdjustFixedReservation(pVM, -(int32_t)cGuestPages, pszDesc);
3418	AssertLogRelMsgStmt(RT_SUCCESS(rc2), ("rc=%Rrc cGuestPages=%#x\n", rc, cGuestPages),
3419	rc = RT_SUCCESS(rc) ? rc2 : rc);
3420	}
3421
3422	/* Are we done? */
3423	if (hMmio2 != NIL_PGMMMIO2HANDLE)
3424	break;
3425	}
3426	}
3427	pgmPhysInvalidatePageMapTLB(pVM);
3428	PGM_UNLOCK(pVM);
3429	return !cFound && hMmio2 != NIL_PGMMMIO2HANDLE ? VERR_NOT_FOUND : rc;
3430	}
3431
3432
3433	/**
3434	* Worker form PGMR3PhysMmio2Map.
3435	*/
3436	static int pgmR3PhysMmio2MapLocked(PVM pVM, uint32_t const idxFirst, uint32_t const cChunks,
3437	RTGCPHYS const GCPhys, RTGCPHYS const GCPhysLast)
3438	{
3439	/*
3440	* Validate the mapped status now that we've got the lock.
3441	*/
3442	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3443	{
3444	AssertReturn( pVM->pgm.s.aMmio2Ranges[idx].GCPhys == NIL_RTGCPHYS
3445	&& !(pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED),
3446	VERR_WRONG_ORDER);
3447	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3448	AssertReturn(pRamRange->GCPhys == NIL_RTGCPHYS, VERR_INTERNAL_ERROR_3);
3449	AssertReturn(pRamRange->GCPhysLast == NIL_RTGCPHYS, VERR_INTERNAL_ERROR_3);
3450	Assert(pRamRange->pbR3 == pVM->pgm.s.aMmio2Ranges[idx].pbR3);
3451	Assert(pRamRange->idRange == pVM->pgm.s.aMmio2Ranges[idx].idRamRange);
3452	}
3453
3454	const char * const pszDesc = pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc;
3455	#ifdef VBOX_WITH_NATIVE_NEM
3456	uint32_t const fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3457	\| (pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3458	? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0);
3459	#endif
3460
3461	/*
3462	* Now, check if this falls into a regular RAM range or if we should use
3463	* the ad-hoc one.
3464	*
3465	* Note! For reasons of simplictly, we're considering the whole MMIO2 area
3466	* here rather than individual chunks.
3467	*/
3468	int rc = VINF_SUCCESS;
3469	uint32_t idxInsert = UINT32_MAX;
3470	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
3471	if (pOverlappingRange)
3472	{
3473	/* Simplification: all within the same range. */
3474	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
3475	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
3476	("%RGp-%RGp (MMIO2/%s) falls partly outside %RGp-%RGp (%s)\n",
3477	GCPhys, GCPhysLast, pszDesc,
3478	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3479	VERR_PGM_RAM_CONFLICT);
3480
3481	/* Check that is isn't an ad hoc range, but a real RAM range. */
3482	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
3483	("%RGp-%RGp (MMIO2/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
3484	GCPhys, GCPhysLast, pszDesc,
3485	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3486	VERR_PGM_RAM_CONFLICT);
3487
3488	/* There can only be one MMIO2 chunk matching here! */
3489	AssertLogRelMsgReturn(cChunks == 1,
3490	("%RGp-%RGp (MMIO2/%s) consists of %u chunks whereas the RAM (%s) somehow doesn't!\n",
3491	GCPhys, GCPhysLast, pszDesc, cChunks, pOverlappingRange->pszDesc),
3492	VERR_PGM_PHYS_MMIO_EX_IPE);
3493
3494	/* Check that it's all RAM pages. */
3495	PCPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
3496	uint32_t const cMmio2Pages = pVM->pgm.s.apMmio2RamRanges[idxFirst]->cb >> GUEST_PAGE_SHIFT;
3497	uint32_t cPagesLeft = cMmio2Pages;
3498	while (cPagesLeft-- > 0)
3499	{
3500	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
3501	("%RGp-%RGp (MMIO2/%s): %RGp is not a RAM page - type=%d desc=%s\n", GCPhys, GCPhysLast,
3502	pszDesc, pOverlappingRange->GCPhys, PGM_PAGE_GET_TYPE(pPage), pOverlappingRange->pszDesc),
3503	VERR_PGM_RAM_CONFLICT);
3504	pPage++;
3505	}
3506
3507	#ifdef VBOX_WITH_PGM_NEM_MODE
3508	/* We cannot mix MMIO2 into a RAM range in simplified memory mode because pOverlappingRange->pbR3 can't point
3509	both at the RAM and MMIO2, so we won't ever write & read from the actual MMIO2 memory if we try. */
3510	AssertLogRelMsgReturn(!VM_IS_NEM_ENABLED(pVM),
3511	("Putting %s at %RGp-%RGp is not possible in NEM mode because existing %RGp-%RGp (%s) mapping\n",
3512	pszDesc, GCPhys, GCPhysLast,
3513	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
3514	VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE);
3515	#endif
3516
3517	/*
3518	* Make all the pages in the range MMIO/ZERO pages, freeing any
3519	* RAM pages currently mapped here. This might not be 100% correct,
3520	* but so what, we do the same from MMIO...
3521	*/
3522	rc = pgmR3PhysFreePageRange(pVM, pOverlappingRange, GCPhys, GCPhysLast, NULL);
3523	AssertRCReturn(rc, rc);
3524
3525	Log(("PGMR3PhysMmio2Map: %RGp-%RGp %s - inside %RGp-%RGp %s\n", GCPhys, GCPhysLast, pszDesc,
3526	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc));
3527
3528	/*
3529	* We're all in for mapping it now. Update the MMIO2 range to reflect it.
3530	*/
3531	pVM->pgm.s.aMmio2Ranges[idxFirst].GCPhys = GCPhys;
3532	pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags \|= PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED;
3533
3534	/*
3535	* Replace the pages in the range.
3536	*/
3537	PPGMPAGE pPageSrc = &pVM->pgm.s.apMmio2RamRanges[idxFirst]->aPages[0];
3538	PPGMPAGE pPageDst = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
3539	cPagesLeft = cMmio2Pages;
3540	while (cPagesLeft-- > 0)
3541	{
3542	Assert(PGM_PAGE_IS_MMIO(pPageDst));
3543
3544	RTHCPHYS const HCPhys = PGM_PAGE_GET_HCPHYS(pPageSrc);
3545	uint32_t const idPage = PGM_PAGE_GET_PAGEID(pPageSrc);
3546	PGM_PAGE_SET_PAGEID(pVM, pPageDst, idPage);
3547	PGM_PAGE_SET_HCPHYS(pVM, pPageDst, HCPhys);
3548	PGM_PAGE_SET_TYPE(pVM, pPageDst, PGMPAGETYPE_MMIO2);
3549	PGM_PAGE_SET_STATE(pVM, pPageDst, PGM_PAGE_STATE_ALLOCATED);
3550	PGM_PAGE_SET_PDE_TYPE(pVM, pPageDst, PGM_PAGE_PDE_TYPE_DONTCARE);
3551	PGM_PAGE_SET_PTE_INDEX(pVM, pPageDst, 0);
3552	PGM_PAGE_SET_TRACKING(pVM, pPageDst, 0);
3553	/* NEM state is not relevant, see VERR_PGM_NOT_SUPPORTED_FOR_NEM_MODE above. */
3554
3555	pVM->pgm.s.cZeroPages--;
3556	pPageSrc++;
3557	pPageDst++;
3558	}
3559
3560	/* Force a PGM pool flush as guest ram references have been changed. */
3561	/** @todo not entirely SMP safe; assuming for now the guest takes
3562	* care of this internally (not touch mapped mmio while changing the
3563	* mapping). */
3564	PVMCPU pVCpu = VMMGetCpu(pVM);
3565	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3566	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3567	}
3568	else
3569	{
3570	/*
3571	* No RAM range, insert the ones prepared during registration.
3572	*/
3573	Log(("PGMR3PhysMmio2Map: %RGp-%RGp %s - no RAM overlap\n", GCPhys, GCPhysLast, pszDesc));
3574	RTGCPHYS GCPhysCur = GCPhys;
3575	uint32_t iChunk = 0;
3576	uint32_t idx = idxFirst;
3577	for (; iChunk < cChunks; iChunk++, idx++)
3578	{
3579	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3580	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3581	Assert(pRamRange->idRange == pMmio2->idRamRange);
3582	Assert(pMmio2->GCPhys == NIL_RTGCPHYS);
3583
3584	#ifdef VBOX_WITH_NATIVE_NEM
3585	/* Tell NEM and get the new NEM state for the pages. */
3586	uint8_t u2NemState = 0;
3587	if (VM_IS_NEM_ENABLED(pVM))
3588	{
3589	rc = NEMR3NotifyPhysMmioExMapEarly(pVM, GCPhysCur, pRamRange->cb, fNemFlags, NULL /pvRam/, pRamRange->pbR3,
3590	&u2NemState, &pRamRange->uNemRange);
3591	AssertLogRelMsgBreak(RT_SUCCESS(rc),
3592	("%RGp LB %RGp fFlags=%#x (%s)\n",
3593	GCPhysCur, pRamRange->cb, pMmio2->fFlags, pRamRange->pszDesc));
3594	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_MAPPED; /* Set this early to indicate that NEM has been notified. */
3595	}
3596	#endif
3597
3598	/* Clear the tracking data of pages we're going to reactivate. */
3599	PPGMPAGE pPageSrc = &pRamRange->aPages[0];
3600	uint32_t cPagesLeft = pRamRange->cb >> GUEST_PAGE_SHIFT;
3601	while (cPagesLeft-- > 0)
3602	{
3603	PGM_PAGE_SET_TRACKING(pVM, pPageSrc, 0);
3604	PGM_PAGE_SET_PTE_INDEX(pVM, pPageSrc, 0);
3605	#ifdef VBOX_WITH_NATIVE_NEM
3606	PGM_PAGE_SET_NEM_STATE(pPageSrc, u2NemState);
3607	#endif
3608	pPageSrc++;
3609	}
3610
3611	/* Insert the RAM range into the lookup table. */
3612	rc = pgmR3PhysRamRangeInsertLookup(pVM, pRamRange, GCPhysCur, &idxInsert);
3613	AssertRCBreak(rc);
3614
3615	/* Mark the range as fully mapped. */
3616	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_OVERLAPPING;
3617	pMmio2->fFlags \|= PGMREGMMIO2RANGE_F_MAPPED;
3618	pMmio2->GCPhys = GCPhysCur;
3619
3620	/* Advance. */
3621	GCPhysCur += pRamRange->cb;
3622	}
3623	if (RT_FAILURE(rc))
3624	{
3625	/*
3626	* Bail out anything we've done so far.
3627	*/
3628	idxInsert -= 1;
3629	do
3630	{
3631	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3632	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3633
3634	#ifdef VBOX_WITH_NATIVE_NEM
3635	if ( VM_IS_NEM_ENABLED(pVM)
3636	&& (pVM->pgm.s.aMmio2Ranges[idx].fFlags & PGMREGMMIO2RANGE_F_MAPPED))
3637	{
3638	uint8_t u2NemState = UINT8_MAX;
3639	NEMR3NotifyPhysMmioExUnmap(pVM, GCPhysCur, pRamRange->cb, fNemFlags, NULL, pRamRange->pbR3,
3640	&u2NemState, &pRamRange->uNemRange);
3641	if (u2NemState != UINT8_MAX)
3642	pgmPhysSetNemStateForPages(pRamRange->aPages, pRamRange->cb >> GUEST_PAGE_SHIFT, u2NemState);
3643	}
3644	#endif
3645	if (pMmio2->GCPhys != NIL_RTGCPHYS)
3646	pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxInsert);
3647
3648	pMmio2->GCPhys = NIL_RTGCPHYS;
3649	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_MAPPED;
3650
3651	idx--;
3652	} while (iChunk-- > 0);
3653	return rc;
3654	}
3655	}
3656
3657	/*
3658	* If the range have dirty page monitoring enabled, enable that.
3659	*
3660	* We ignore failures here for now because if we fail, the whole mapping
3661	* will have to be reversed and we'll end up with nothing at all on the
3662	* screen and a grumpy guest, whereas if we just go on, we'll only have
3663	* visual distortions to gripe about. There will be something in the
3664	* release log.
3665	*/
3666	if ( pVM->pgm.s.aMmio2Ranges[idxFirst].pPhysHandlerR3
3667	&& (pVM->pgm.s.aMmio2Ranges[idxFirst].fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3668	pgmR3PhysMmio2EnableDirtyPageTracing(pVM, idxFirst, cChunks);
3669
3670	/* Flush physical page map TLB. */
3671	pgmPhysInvalidatePageMapTLB(pVM);
3672
3673	#ifdef VBOX_WITH_NATIVE_NEM
3674	/*
3675	* Late NEM notification (currently unused).
3676	*/
3677	if (VM_IS_NEM_ENABLED(pVM))
3678	{
3679	if (pOverlappingRange)
3680	{
3681	uint8_t * const pbRam = pOverlappingRange->pbR3 ? &pOverlappingRange->pbR3[GCPhys - pOverlappingRange->GCPhys] : NULL;
3682	rc = NEMR3NotifyPhysMmioExMapLate(pVM, GCPhys, GCPhysLast - GCPhys + 1U,
3683	fNemFlags \| NEM_NOTIFY_PHYS_MMIO_EX_F_REPLACE, pbRam,
3684	pVM->pgm.s.aMmio2Ranges[idxFirst].pbR3, NULL /puNemRange/);
3685	}
3686	else
3687	{
3688	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3689	{
3690	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3691	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3692	Assert(pMmio2->GCPhys == pRamRange->GCPhys);
3693
3694	rc = NEMR3NotifyPhysMmioExMapLate(pVM, pRamRange->GCPhys, pRamRange->cb, fNemFlags, NULL /pvRam/,
3695	pRamRange->pbR3, &pRamRange->uNemRange);
3696	AssertRCBreak(rc);
3697	}
3698	}
3699	AssertLogRelRCReturnStmt(rc,
3700	PGMR3PhysMmio2Unmap(pVM, pVM->pgm.s.aMmio2Ranges[idxFirst].pDevInsR3, idxFirst + 1, GCPhys),
3701	rc);
3702	}
3703	#endif
3704
3705	return VINF_SUCCESS;
3706	}
3707
3708
3709	/**
3710	* Maps a MMIO2 region.
3711	*
3712	* This is typically done when a guest / the bios / state loading changes the
3713	* PCI config. The replacing of base memory has the same restrictions as during
3714	* registration, of course.
3715	*
3716	* @returns VBox status code.
3717	*
3718	* @param pVM The cross context VM structure.
3719	* @param pDevIns The device instance owning the region.
3720	* @param hMmio2 The handle of the region to map.
3721	* @param GCPhys The guest-physical address to be remapped.
3722	*/
3723	VMMR3_INT_DECL(int) PGMR3PhysMmio2Map(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3724	{
3725	/*
3726	* Validate input.
3727	*/
3728	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3729	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3730	AssertReturn(GCPhys != NIL_RTGCPHYS, VERR_INVALID_PARAMETER);
3731	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3732	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3733	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3734
3735	uint32_t cChunks = 0;
3736	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
3737	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
3738
3739	/* Gather the full range size so we can validate the mapping address properly. */
3740	RTGCPHYS cbRange = 0;
3741	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3742	cbRange += pVM->pgm.s.apMmio2RamRanges[idx]->cb;
3743
3744	RTGCPHYS const GCPhysLast = GCPhys + cbRange - 1;
3745	AssertLogRelReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
3746
3747	/*
3748	* Take the PGM lock and call worker.
3749	*/
3750	int rc = PGM_LOCK(pVM);
3751	AssertRCReturn(rc, rc);
3752
3753	rc = pgmR3PhysMmio2MapLocked(pVM, idxFirst, cChunks, GCPhys, GCPhysLast);
3754	#ifdef VBOX_STRICT
3755	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
3756	#endif
3757
3758	PGM_UNLOCK(pVM);
3759	return rc;
3760	}
3761
3762
3763	/**
3764	* Worker form PGMR3PhysMmio2Map.
3765	*/
3766	static int pgmR3PhysMmio2UnmapLocked(PVM pVM, uint32_t const idxFirst, uint32_t const cChunks, RTGCPHYS const GCPhysIn)
3767	{
3768	/*
3769	* Validate input.
3770	*/
3771	RTGCPHYS cbRange = 0;
3772	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3773	{
3774	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3775	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3776	AssertReturn(pMmio2->idRamRange == pRamRange->idRange, VERR_INTERNAL_ERROR_3);
3777	AssertReturn(pMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED, VERR_WRONG_ORDER);
3778	AssertReturn(pMmio2->GCPhys != NIL_RTGCPHYS, VERR_WRONG_ORDER);
3779	cbRange += pRamRange->cb;
3780	}
3781
3782	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
3783	PPGMRAMRANGE const pFirstRamRange = pVM->pgm.s.apMmio2RamRanges[idxFirst];
3784	const char * const pszDesc = pFirstRamRange->pszDesc;
3785	AssertLogRelMsgReturn(GCPhysIn == pFirstMmio2->GCPhys \|\| GCPhysIn == NIL_RTGCPHYS,
3786	("GCPhys=%RGp, actual address is %RGp\n", GCPhysIn, pFirstMmio2->GCPhys),
3787	VERR_MISMATCH);
3788	RTGCPHYS const GCPhys = pFirstMmio2->GCPhys; /* (it's always NIL_RTGCPHYS) */
3789	Log(("PGMR3PhysMmio2Unmap: %RGp-%RGp %s\n", GCPhys, GCPhys + cbRange - 1U, pszDesc));
3790
3791	uint16_t const fOldFlags = pFirstMmio2->fFlags;
3792	Assert(fOldFlags & PGMREGMMIO2RANGE_F_MAPPED);
3793
3794	/* Find the first entry in the lookup table and verify the overlapping flag. */
3795	uint32_t idxLookup = pgmR3PhysRamRangeFindOverlappingIndex(pVM, GCPhys, GCPhys + pFirstRamRange->cb - 1U);
3796	AssertLogRelMsgReturn(idxLookup < pVM->pgm.s.RamRangeUnion.cLookupEntries,
3797	("MMIO2 range not found at %RGp LB %RGp in the lookup table! (%s)\n",
3798	GCPhys, pFirstRamRange->cb, pszDesc),
3799	VERR_INTERNAL_ERROR_2);
3800
3801	uint32_t const idLookupRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
3802	AssertLogRelReturn(idLookupRange != 0 && idLookupRange <= pVM->pgm.s.idRamRangeMax, VERR_INTERNAL_ERROR_5);
3803	PPGMRAMRANGE const pLookupRange = pVM->pgm.s.apRamRanges[idLookupRange];
3804	AssertLogRelReturn(pLookupRange, VERR_INTERNAL_ERROR_3);
3805
3806	AssertLogRelMsgReturn(fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING
3807	? pLookupRange != pFirstRamRange : pLookupRange == pFirstRamRange,
3808	("MMIO2 unmap mixup at %RGp LB %RGp fl=%#x (%s) vs %RGp LB %RGp (%s)\n",
3809	GCPhys, cbRange, fOldFlags, pszDesc, pLookupRange->GCPhys, pLookupRange->cb, pLookupRange->pszDesc),
3810	VERR_INTERNAL_ERROR_4);
3811
3812	/*
3813	* If monitoring dirty pages, we must deregister the handlers first.
3814	*/
3815	if ( pFirstMmio2->pPhysHandlerR3
3816	&& (fOldFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
3817	pgmR3PhysMmio2DisableDirtyPageTracing(pVM, idxFirst, cChunks);
3818
3819	/*
3820	* Unmap it.
3821	*/
3822	int rcRet = VINF_SUCCESS;
3823	#ifdef VBOX_WITH_NATIVE_NEM
3824	uint32_t const fNemFlags = NEM_NOTIFY_PHYS_MMIO_EX_F_MMIO2
3825	\| (fOldFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES
3826	? NEM_NOTIFY_PHYS_MMIO_EX_F_TRACK_DIRTY_PAGES : 0);
3827	#endif
3828	if (fOldFlags & PGMREGMMIO2RANGE_F_OVERLAPPING)
3829	{
3830	/*
3831	* We've replaced RAM, replace with zero pages.
3832	*
3833	* Note! This is where we might differ a little from a real system, because
3834	* it's likely to just show the RAM pages as they were before the
3835	* MMIO2 region was mapped here.
3836	*/
3837	/* Only one chunk allowed when overlapping! */
3838	Assert(cChunks == 1);
3839	/* No NEM stuff should ever get here, see assertion in the mapping function. */
3840	AssertReturn(!VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
3841
3842	/* Restore the RAM pages we've replaced. */
3843	PPGMPAGE pPageDst = &pLookupRange->aPages[(pFirstRamRange->GCPhys - pLookupRange->GCPhys) >> GUEST_PAGE_SHIFT];
3844	uint32_t cPagesLeft = pFirstRamRange->cb >> GUEST_PAGE_SHIFT;
3845	pVM->pgm.s.cZeroPages += cPagesLeft;
3846	while (cPagesLeft-- > 0)
3847	{
3848	PGM_PAGE_INIT_ZERO(pPageDst, pVM, PGMPAGETYPE_RAM);
3849	pPageDst++;
3850	}
3851
3852	/* Update range state. */
3853	pFirstMmio2->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED);
3854	pFirstMmio2->GCPhys = NIL_RTGCPHYS;
3855	Assert(pFirstRamRange->GCPhys == NIL_RTGCPHYS);
3856	Assert(pFirstRamRange->GCPhysLast == NIL_RTGCPHYS);
3857	}
3858	else
3859	{
3860	/*
3861	* Unlink the chunks related to the MMIO/MMIO2 region.
3862	*/
3863	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
3864	{
3865	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
3866	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
3867	Assert(pMmio2->idRamRange == pRamRange->idRange);
3868	Assert(pMmio2->GCPhys == pRamRange->GCPhys);
3869
3870	#ifdef VBOX_WITH_NATIVE_NEM
3871	if (VM_IS_NEM_ENABLED(pVM)) /* Notify NEM. */
3872	{
3873	uint8_t u2State = UINT8_MAX;
3874	int rc = NEMR3NotifyPhysMmioExUnmap(pVM, pRamRange->GCPhys, pRamRange->cb, fNemFlags,
3875	NULL, pMmio2->pbR3, &u2State, &pRamRange->uNemRange);
3876	AssertLogRelMsgStmt(RT_SUCCESS(rc),
3877	("NEMR3NotifyPhysMmioExUnmap failed: %Rrc - GCPhys=RGp LB %RGp fNemFlags=%#x pbR3=%p %s\n",
3878	rc, pRamRange->GCPhys, pRamRange->cb, fNemFlags, pMmio2->pbR3, pRamRange->pszDesc),
3879	rcRet = rc);
3880	if (u2State != UINT8_MAX)
3881	pgmPhysSetNemStateForPages(pRamRange->aPages, pRamRange->cb >> GUEST_PAGE_SHIFT, u2State);
3882	}
3883	#endif
3884
3885	int rc = pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxLookup);
3886	AssertLogRelMsgStmt(RT_SUCCESS(rc),
3887	("pgmR3PhysRamRangeRemoveLookup failed: %Rrc - GCPhys=%RGp LB %RGp %s\n",
3888	rc, pRamRange->GCPhys, pRamRange->cb, pRamRange->pszDesc),
3889	rcRet = rc);
3890
3891	pMmio2->GCPhys = NIL_RTGCPHYS;
3892	pMmio2->fFlags &= ~(PGMREGMMIO2RANGE_F_OVERLAPPING \| PGMREGMMIO2RANGE_F_MAPPED);
3893	Assert(pRamRange->GCPhys == NIL_RTGCPHYS);
3894	Assert(pRamRange->GCPhysLast == NIL_RTGCPHYS);
3895	}
3896	}
3897
3898	/* Force a PGM pool flush as guest ram references have been changed. */
3899	/** @todo not entirely SMP safe; assuming for now the guest takes care
3900	* of this internally (not touch mapped mmio while changing the
3901	* mapping). */
3902	PVMCPU pVCpu = VMMGetCpu(pVM);
3903	pVCpu->pgm.s.fSyncFlags \|= PGM_SYNC_CLEAR_PGM_POOL;
3904	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
3905
3906	pgmPhysInvalidatePageMapTLB(pVM);
3907	pgmPhysInvalidRamRangeTlbs(pVM);
3908
3909	return rcRet;
3910	}
3911
3912
3913	/**
3914	* Unmaps an MMIO2 region.
3915	*
3916	* This is typically done when a guest / the bios / state loading changes the
3917	* PCI config. The replacing of base memory has the same restrictions as during
3918	* registration, of course.
3919	*/
3920	VMMR3_INT_DECL(int) PGMR3PhysMmio2Unmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS GCPhys)
3921	{
3922	/*
3923	* Validate input
3924	*/
3925	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
3926	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3927	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE, VERR_INVALID_HANDLE);
3928	if (GCPhys != NIL_RTGCPHYS)
3929	{
3930	AssertReturn(GCPhys != 0, VERR_INVALID_PARAMETER);
3931	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
3932	}
3933
3934	uint32_t cChunks = 0;
3935	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
3936	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
3937
3938
3939	/*
3940	* Take the PGM lock and call worker.
3941	*/
3942	int rc = PGM_LOCK(pVM);
3943	AssertRCReturn(rc, rc);
3944
3945	rc = pgmR3PhysMmio2UnmapLocked(pVM, idxFirst, cChunks, GCPhys);
3946	#ifdef VBOX_STRICT
3947	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
3948	#endif
3949
3950	PGM_UNLOCK(pVM);
3951	return rc;
3952	}
3953
3954
3955	/**
3956	* Reduces the mapping size of a MMIO2 region.
3957	*
3958	* This is mainly for dealing with old saved states after changing the default
3959	* size of a mapping region. See PDMDevHlpMmio2Reduce and
3960	* PDMPCIDEV::pfnRegionLoadChangeHookR3.
3961	*
3962	* The region must not currently be mapped when making this call. The VM state
3963	* must be state restore or VM construction.
3964	*
3965	* @returns VBox status code.
3966	* @param pVM The cross context VM structure.
3967	* @param pDevIns The device instance owning the region.
3968	* @param hMmio2 The handle of the region to reduce.
3969	* @param cbRegion The new mapping size.
3970	*/
3971	VMMR3_INT_DECL(int) PGMR3PhysMmio2Reduce(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, RTGCPHYS cbRegion)
3972	{
3973	/*
3974	* Validate input
3975	*/
3976	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
3977	AssertReturn(hMmio2 != NIL_PGMMMIO2HANDLE && hMmio2 != 0 && hMmio2 <= RT_ELEMENTS(pVM->pgm.s.aMmio2Ranges),
3978	VERR_INVALID_HANDLE);
3979	AssertReturn(cbRegion >= GUEST_PAGE_SIZE, VERR_INVALID_PARAMETER);
3980	AssertReturn(!(cbRegion & GUEST_PAGE_OFFSET_MASK), VERR_UNSUPPORTED_ALIGNMENT);
3981
3982	PVMCPU const pVCpu = VMMGetCpu(pVM);
3983	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
3984
3985	VMSTATE const enmVmState = VMR3GetState(pVM);
3986	AssertLogRelMsgReturn( enmVmState == VMSTATE_CREATING
3987	\|\| enmVmState == VMSTATE_LOADING,
3988	("enmVmState=%d (%s)\n", enmVmState, VMR3GetStateName(enmVmState)),
3989	VERR_VM_INVALID_VM_STATE);
3990
3991	/*
3992	* Grab the PGM lock and validate the request properly.
3993	*/
3994	int rc = PGM_LOCK(pVM);
3995	AssertRCReturn(rc, rc);
3996
3997	uint32_t cChunks = 0;
3998	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
3999	if ((int32_t)idxFirst >= 0)
4000	{
4001	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4002	PPGMRAMRANGE const pFirstRamRange = pVM->pgm.s.apMmio2RamRanges[idxFirst];
4003	if ( !(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
4004	&& pFirstMmio2->GCPhys == NIL_RTGCPHYS)
4005	{
4006	/*
4007	* NOTE! Current implementation does not support multiple ranges.
4008	* Implement when there is a real world need and thus a testcase.
4009	*/
4010	if (cChunks == 1)
4011	{
4012	/*
4013	* The request has to be within the initial size.
4014	*/
4015	if (cbRegion <= pFirstMmio2->cbReal)
4016	{
4017	/*
4018	* All we have to do is modify the size stored in the RAM range,
4019	* as it is the one used when mapping it and such.
4020	* The two page counts stored in PGMR0PERVM remain unchanged.
4021	*/
4022	Log(("PGMR3PhysMmio2Reduce: %s changes from %#RGp bytes (%#RGp) to %#RGp bytes.\n",
4023	pFirstRamRange->pszDesc, pFirstRamRange->cb, pFirstMmio2->cbReal, cbRegion));
4024	pFirstRamRange->cb = cbRegion;
4025	rc = VINF_SUCCESS;
4026	}
4027	else
4028	{
4029	AssertLogRelMsgFailed(("MMIO2/%s: cbRegion=%#RGp > cbReal=%#RGp\n",
4030	pFirstRamRange->pszDesc, cbRegion, pFirstMmio2->cbReal));
4031	rc = VERR_OUT_OF_RANGE;
4032	}
4033	}
4034	else
4035	{
4036	AssertLogRelMsgFailed(("MMIO2/%s: more than one chunk: %d (flags=%#x)\n",
4037	pFirstRamRange->pszDesc, cChunks, pFirstMmio2->fFlags));
4038	rc = VERR_NOT_SUPPORTED;
4039	}
4040	}
4041	else
4042	{
4043	AssertLogRelMsgFailed(("MMIO2/%s: cannot change size of mapped range: %RGp..%RGp\n", pFirstRamRange->pszDesc,
4044	pFirstMmio2->GCPhys, pFirstMmio2->GCPhys + pFirstRamRange->cb - 1U));
4045	rc = VERR_WRONG_ORDER;
4046	}
4047	}
4048	else
4049	rc = (int32_t)idxFirst;
4050
4051	PGM_UNLOCK(pVM);
4052	return rc;
4053	}
4054
4055
4056	/**
4057	* Validates @a hMmio2, making sure it belongs to @a pDevIns.
4058	*
4059	* @returns VBox status code.
4060	* @param pVM The cross context VM structure.
4061	* @param pDevIns The device which allegedly owns @a hMmio2.
4062	* @param hMmio2 The handle to validate.
4063	*/
4064	VMMR3_INT_DECL(int) PGMR3PhysMmio2ValidateHandle(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
4065	{
4066	/*
4067	* Validate input
4068	*/
4069	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4070	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4071
4072	/*
4073	* Just do this the simple way.
4074	*/
4075	int rc = PGM_LOCK_VOID(pVM);
4076	AssertRCReturn(rc, rc);
4077	uint32_t cChunks;
4078	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4079	PGM_UNLOCK(pVM);
4080	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4081	return VINF_SUCCESS;
4082	}
4083
4084
4085	/**
4086	* Gets the mapping address of an MMIO2 region.
4087	*
4088	* @returns Mapping address, NIL_RTGCPHYS if not mapped or invalid handle.
4089	*
4090	* @param pVM The cross context VM structure.
4091	* @param pDevIns The device owning the MMIO2 handle.
4092	* @param hMmio2 The region handle.
4093	*/
4094	VMMR3_INT_DECL(RTGCPHYS) PGMR3PhysMmio2GetMappingAddress(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2)
4095	{
4096	RTGCPHYS GCPhysRet = NIL_RTGCPHYS;
4097
4098	int rc = PGM_LOCK_VOID(pVM);
4099	AssertRCReturn(rc, NIL_RTGCPHYS);
4100
4101	uint32_t cChunks;
4102	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4103	if ((int32_t)idxFirst >= 0)
4104	GCPhysRet = pVM->pgm.s.aMmio2Ranges[idxFirst].GCPhys;
4105
4106	PGM_UNLOCK(pVM);
4107	return NIL_RTGCPHYS;
4108	}
4109
4110
4111	/**
4112	* Worker for PGMR3PhysMmio2QueryAndResetDirtyBitmap.
4113	*
4114	* Called holding the PGM lock.
4115	*/
4116	static int pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
4117	void *pvBitmap, size_t cbBitmap)
4118	{
4119	/*
4120	* Continue validation.
4121	*/
4122	uint32_t cChunks;
4123	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4124	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4125	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4126	AssertReturn(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES, VERR_INVALID_FUNCTION);
4127
4128	int rc = VINF_SUCCESS;
4129	if (cbBitmap \|\| pvBitmap)
4130	{
4131	/*
4132	* Check the bitmap size and collect all the dirty flags.
4133	*/
4134	RTGCPHYS cbTotal = 0;
4135	uint16_t fTotalDirty = 0;
4136	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4137	{
4138	/* Not using cbReal here, because NEM is not in on the creating, only the mapping. */
4139	cbTotal += pVM->pgm.s.apMmio2RamRanges[idx]->cb;
4140	fTotalDirty \|= pVM->pgm.s.aMmio2Ranges[idx].fFlags;
4141	}
4142	size_t const cbTotalBitmap = RT_ALIGN_T(cbTotal, GUEST_PAGE_SIZE * 64, RTGCPHYS) / GUEST_PAGE_SIZE / 8;
4143
4144	AssertPtrReturn(pvBitmap, VERR_INVALID_POINTER);
4145	AssertReturn(RT_ALIGN_P(pvBitmap, sizeof(uint64_t)) == pvBitmap, VERR_INVALID_POINTER);
4146	AssertReturn(cbBitmap == cbTotalBitmap, VERR_INVALID_PARAMETER);
4147
4148	#ifdef VBOX_WITH_PGM_NEM_MODE
4149	/*
4150	* If there is no physical handler we must be in NEM mode and NEM
4151	* taking care of the dirty bit collecting.
4152	*/
4153	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4154	{
4155	/** @todo This does not integrate at all with --execute-all-in-iem, leaving the
4156	* screen blank when using it together with --driverless. Fixing this won't be
4157	* entirely easy as we take the PGM_PAGE_HNDL_PHYS_STATE_DISABLED page status to
4158	* mean a dirty page. */
4159	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4160	uint8_t pbBitmap = (uint8_t )pvBitmap;
4161	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4162	{
4163	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4164	size_t const cbBitmapChunk = (pRamRange->cb / GUEST_PAGE_SIZE + 7) / 8;
4165	Assert((RTGCPHYS)cbBitmapChunk * GUEST_PAGE_SIZE * 8 == pRamRange->cb);
4166	Assert(pRamRange->GCPhys == pVM->pgm.s.aMmio2Ranges[idx].GCPhys); /* (No MMIO2 inside RAM in NEM mode!)*/
4167	int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pRamRange->GCPhys, pRamRange->cb,
4168	pRamRange->uNemRange, pbBitmap, cbBitmapChunk);
4169	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4170	rc = rc2;
4171	pbBitmap += pRamRange->cb / GUEST_PAGE_SIZE / 8;
4172	}
4173	}
4174	else
4175	#endif
4176	if (fTotalDirty & PGMREGMMIO2RANGE_F_IS_DIRTY)
4177	{
4178	if ( (pFirstMmio2->fFlags & (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4179	== (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4180	{
4181	/*
4182	* Reset each chunk, gathering dirty bits.
4183	*/
4184	RT_BZERO(pvBitmap, cbBitmap); /* simpler for now. */
4185	for (uint32_t iChunk = 0, idx = idxFirst, iPageNo = 0; iChunk < cChunks; iChunk++, idx++)
4186	{
4187	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
4188	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
4189	{
4190	int rc2 = pgmHandlerPhysicalResetMmio2WithBitmap(pVM, pMmio2->GCPhys, pvBitmap, iPageNo);
4191	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4192	rc = rc2;
4193	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4194	}
4195	iPageNo += pVM->pgm.s.apMmio2RamRanges[idx]->cb >> GUEST_PAGE_SHIFT;
4196	}
4197	}
4198	else
4199	{
4200	/*
4201	* If not mapped or tracking is disabled, we return the
4202	* PGMREGMMIO2RANGE_F_IS_DIRTY status for all pages. We cannot
4203	* get more accurate data than that after unmapping or disabling.
4204	*/
4205	RT_BZERO(pvBitmap, cbBitmap);
4206	for (uint32_t iChunk = 0, idx = idxFirst, iPageNo = 0; iChunk < cChunks; iChunk++, idx++)
4207	{
4208	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4209	PPGMREGMMIO2RANGE const pMmio2 = &pVM->pgm.s.aMmio2Ranges[idx];
4210	if (pMmio2->fFlags & PGMREGMMIO2RANGE_F_IS_DIRTY)
4211	{
4212	ASMBitSetRange(pvBitmap, iPageNo, iPageNo + (pRamRange->cb >> GUEST_PAGE_SHIFT));
4213	pMmio2->fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4214	}
4215	iPageNo += pRamRange->cb >> GUEST_PAGE_SHIFT;
4216	}
4217	}
4218	}
4219	/*
4220	* No dirty chunks.
4221	*/
4222	else
4223	RT_BZERO(pvBitmap, cbBitmap);
4224	}
4225	/*
4226	* No bitmap. Reset the region if tracking is currently enabled.
4227	*/
4228	else if ( (pFirstMmio2->fFlags & (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4229	== (PGMREGMMIO2RANGE_F_MAPPED \| PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4230	{
4231	#ifdef VBOX_WITH_PGM_NEM_MODE
4232	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4233	{
4234	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4235	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4236	{
4237	PPGMRAMRANGE const pRamRange = pVM->pgm.s.apMmio2RamRanges[idx];
4238	Assert(pRamRange->GCPhys == pVM->pgm.s.aMmio2Ranges[idx].GCPhys); /* (No MMIO2 inside RAM in NEM mode!)*/
4239	int rc2 = NEMR3PhysMmio2QueryAndResetDirtyBitmap(pVM, pRamRange->GCPhys, pRamRange->cb,
4240	pRamRange->uNemRange, NULL, 0);
4241	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4242	rc = rc2;
4243	}
4244	}
4245	else
4246	#endif
4247	{
4248	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4249	{
4250	pVM->pgm.s.aMmio2Ranges[idx].fFlags &= ~PGMREGMMIO2RANGE_F_IS_DIRTY;
4251	int rc2 = PGMHandlerPhysicalReset(pVM, pVM->pgm.s.aMmio2Ranges[idx].GCPhys);
4252	if (RT_FAILURE(rc2) && RT_SUCCESS(rc))
4253	rc = rc2;
4254	}
4255	}
4256	}
4257
4258	return rc;
4259	}
4260
4261
4262	/**
4263	* Queries the dirty page bitmap and resets the monitoring.
4264	*
4265	* The PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES flag must be specified when
4266	* creating the range for this to work.
4267	*
4268	* @returns VBox status code.
4269	* @retval VERR_INVALID_FUNCTION if not created using
4270	* PGMPHYS_MMIO2_FLAGS_TRACK_DIRTY_PAGES.
4271	* @param pVM The cross context VM structure.
4272	* @param pDevIns The device owning the MMIO2 handle.
4273	* @param hMmio2 The region handle.
4274	* @param pvBitmap The output bitmap. Must be 8-byte aligned. Ignored
4275	* when @a cbBitmap is zero.
4276	* @param cbBitmap The size of the bitmap. Must be the size of the whole
4277	* MMIO2 range, rounded up to the nearest 8 bytes.
4278	* When zero only a reset is done.
4279	*/
4280	VMMR3_INT_DECL(int) PGMR3PhysMmio2QueryAndResetDirtyBitmap(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2,
4281	void *pvBitmap, size_t cbBitmap)
4282	{
4283	/*
4284	* Do some basic validation before grapping the PGM lock and continuing.
4285	*/
4286	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4287	AssertReturn(RT_ALIGN_Z(cbBitmap, sizeof(uint64_t)) == cbBitmap, VERR_INVALID_PARAMETER);
4288	int rc = PGM_LOCK(pVM);
4289	if (RT_SUCCESS(rc))
4290	{
4291	STAM_PROFILE_START(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4292	rc = pgmR3PhysMmio2QueryAndResetDirtyBitmapLocked(pVM, pDevIns, hMmio2, pvBitmap, cbBitmap);
4293	STAM_PROFILE_STOP(&pVM->pgm.s.StatMmio2QueryAndResetDirtyBitmap, a);
4294	PGM_UNLOCK(pVM);
4295	}
4296	return rc;
4297	}
4298
4299
4300	/**
4301	* Worker for PGMR3PhysMmio2ControlDirtyPageTracking
4302	*
4303	* Called owning the PGM lock.
4304	*/
4305	static int pgmR3PhysMmio2ControlDirtyPageTrackingLocked(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4306	{
4307	/*
4308	* Continue validation.
4309	*/
4310	uint32_t cChunks;
4311	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4312	AssertReturn((int32_t)idxFirst >= 0, (int32_t)idxFirst);
4313	PPGMREGMMIO2RANGE const pFirstMmio2 = &pVM->pgm.s.aMmio2Ranges[idxFirst];
4314	AssertReturn(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACK_DIRTY_PAGES, VERR_INVALID_FUNCTION);
4315
4316	#ifdef VBOX_WITH_PGM_NEM_MODE
4317	/*
4318	* This is a nop if NEM is responsible for doing the tracking, we simply
4319	* leave the tracking on all the time there.
4320	*/
4321	if (pFirstMmio2->pPhysHandlerR3 == NULL)
4322	{
4323	AssertReturn(VM_IS_NEM_ENABLED(pVM), VERR_INTERNAL_ERROR_4);
4324	return VINF_SUCCESS;
4325	}
4326	#endif
4327
4328	/*
4329	* Anything needing doing?
4330	*/
4331	if (fEnabled != RT_BOOL(pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_TRACKING_ENABLED))
4332	{
4333	LogFlowFunc(("fEnabled=%RTbool %s\n", fEnabled, pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc));
4334
4335	/*
4336	* Update the PGMREGMMIO2RANGE_F_TRACKING_ENABLED flag.
4337	*/
4338	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4339	if (fEnabled)
4340	pVM->pgm.s.aMmio2Ranges[idx].fFlags \|= PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4341	else
4342	pVM->pgm.s.aMmio2Ranges[idx].fFlags &= ~PGMREGMMIO2RANGE_F_TRACKING_ENABLED;
4343
4344	/*
4345	* Enable/disable handlers if currently mapped.
4346	*
4347	* We ignore status codes here as we've already changed the flags and
4348	* returning a failure status now would be confusing. Besides, the two
4349	* functions will continue past failures. As argued in the mapping code,
4350	* it's in the release log.
4351	*/
4352	if (pFirstMmio2->fFlags & PGMREGMMIO2RANGE_F_MAPPED)
4353	{
4354	if (fEnabled)
4355	pgmR3PhysMmio2EnableDirtyPageTracing(pVM, idxFirst, cChunks);
4356	else
4357	pgmR3PhysMmio2DisableDirtyPageTracing(pVM, idxFirst, cChunks);
4358	}
4359	}
4360	else
4361	LogFlowFunc(("fEnabled=%RTbool %s - no change\n", fEnabled, pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc));
4362
4363	return VINF_SUCCESS;
4364	}
4365
4366
4367	/**
4368	* Controls the dirty page tracking for an MMIO2 range.
4369	*
4370	* @returns VBox status code.
4371	* @param pVM The cross context VM structure.
4372	* @param pDevIns The device owning the MMIO2 memory.
4373	* @param hMmio2 The handle of the region.
4374	* @param fEnabled The new tracking state.
4375	*/
4376	VMMR3_INT_DECL(int) PGMR3PhysMmio2ControlDirtyPageTracking(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, bool fEnabled)
4377	{
4378	/*
4379	* Do some basic validation before grapping the PGM lock and continuing.
4380	*/
4381	AssertPtrReturn(pDevIns, VERR_INVALID_POINTER);
4382	int rc = PGM_LOCK(pVM);
4383	if (RT_SUCCESS(rc))
4384	{
4385	rc = pgmR3PhysMmio2ControlDirtyPageTrackingLocked(pVM, pDevIns, hMmio2, fEnabled);
4386	PGM_UNLOCK(pVM);
4387	}
4388	return rc;
4389	}
4390
4391
4392	/**
4393	* Changes the region number of an MMIO2 region.
4394	*
4395	* This is only for dealing with save state issues, nothing else.
4396	*
4397	* @return VBox status code.
4398	*
4399	* @param pVM The cross context VM structure.
4400	* @param pDevIns The device owning the MMIO2 memory.
4401	* @param hMmio2 The handle of the region.
4402	* @param iNewRegion The new region index.
4403	*
4404	* @thread EMT(0)
4405	* @sa @bugref{9359}
4406	*/
4407	VMMR3_INT_DECL(int) PGMR3PhysMmio2ChangeRegionNo(PVM pVM, PPDMDEVINS pDevIns, PGMMMIO2HANDLE hMmio2, uint32_t iNewRegion)
4408	{
4409	/*
4410	* Validate input.
4411	*/
4412	VM_ASSERT_EMT0_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
4413	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_LOADING, VERR_VM_INVALID_VM_STATE);
4414	AssertReturn(iNewRegion <= UINT8_MAX, VERR_INVALID_PARAMETER);
4415
4416	int rc = PGM_LOCK(pVM);
4417	AssertRCReturn(rc, rc);
4418
4419	/* Validate and resolve the handle. */
4420	uint32_t cChunks;
4421	uint32_t const idxFirst = pgmR3PhysMmio2ResolveHandle(pVM, pDevIns, hMmio2, &cChunks);
4422	if ((int32_t)idxFirst >= 0)
4423	{
4424	/* Check that the new range number is unused. */
4425	PPGMREGMMIO2RANGE const pConflict = pgmR3PhysMmio2Find(pVM, pDevIns, pVM->pgm.s.aMmio2Ranges[idxFirst].iSubDev,
4426	iNewRegion);
4427	if (!pConflict)
4428	{
4429	/*
4430	* Make the change.
4431	*/
4432	for (uint32_t iChunk = 0, idx = idxFirst; iChunk < cChunks; iChunk++, idx++)
4433	pVM->pgm.s.aMmio2Ranges[idx].iRegion = (uint8_t)iNewRegion;
4434	rc = VINF_SUCCESS;
4435	}
4436	else
4437	{
4438	AssertLogRelMsgFailed(("MMIO2/%s: iNewRegion=%d conflicts with %s\n", pVM->pgm.s.apMmio2RamRanges[idxFirst]->pszDesc,
4439	iNewRegion, pVM->pgm.s.apMmio2RamRanges[pConflict->idRamRange]->pszDesc));
4440	rc = VERR_RESOURCE_IN_USE;
4441	}
4442	}
4443	else
4444	rc = (int32_t)idxFirst;
4445
4446	PGM_UNLOCK(pVM);
4447	return rc;
4448	}
4449
4450
4451
4452	/*********************************************************************************************************************************
4453	* ROM *
4454	*********************************************************************************************************************************/
4455
4456	/**
4457	* Worker for PGMR3PhysRomRegister.
4458	*
4459	* This is here to simplify lock management, i.e. the caller does all the
4460	* locking and we can simply return without needing to remember to unlock
4461	* anything first.
4462	*
4463	* @returns VBox status code.
4464	* @param pVM The cross context VM structure.
4465	* @param pDevIns The device instance owning the ROM.
4466	* @param GCPhys First physical address in the range.
4467	* Must be page aligned!
4468	* @param cb The size of the range (in bytes).
4469	* Must be page aligned!
4470	* @param pvBinary Pointer to the binary data backing the ROM image.
4471	* @param cbBinary The size of the binary data pvBinary points to.
4472	* This must be less or equal to @a cb.
4473	* @param fFlags Mask of flags. PGMPHYS_ROM_FLAGS_SHADOWED
4474	* and/or PGMPHYS_ROM_FLAGS_PERMANENT_BINARY.
4475	* @param pszDesc Pointer to description string. This must not be freed.
4476	*/
4477	static int pgmR3PhysRomRegisterLocked(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4478	const void pvBinary, uint32_t cbBinary, uint8_t fFlags, const char pszDesc)
4479	{
4480	/*
4481	* Validate input.
4482	*/
4483	AssertPtrReturn(pDevIns, VERR_INVALID_PARAMETER);
4484	AssertReturn(RT_ALIGN_T(GCPhys, GUEST_PAGE_SIZE, RTGCPHYS) == GCPhys, VERR_INVALID_PARAMETER);
4485	AssertReturn(RT_ALIGN_T(cb, GUEST_PAGE_SIZE, RTGCPHYS) == cb, VERR_INVALID_PARAMETER);
4486	RTGCPHYS const GCPhysLast = GCPhys + (cb - 1);
4487	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
4488	AssertPtrReturn(pvBinary, VERR_INVALID_PARAMETER);
4489	AssertPtrReturn(pszDesc, VERR_INVALID_POINTER);
4490	AssertReturn(!(fFlags & ~PGMPHYS_ROM_FLAGS_VALID_MASK), VERR_INVALID_PARAMETER);
4491
4492	PVMCPU const pVCpu = VMMGetCpu(pVM);
4493	AssertReturn(pVCpu && pVCpu->idCpu == 0, VERR_VM_THREAD_NOT_EMT);
4494	VM_ASSERT_STATE_RETURN(pVM, VMSTATE_CREATING, VERR_VM_INVALID_VM_STATE);
4495
4496	const uint32_t cGuestPages = cb >> GUEST_PAGE_SHIFT;
4497	AssertReturn(cGuestPages <= PGM_MAX_PAGES_PER_ROM_RANGE, VERR_OUT_OF_RANGE);
4498
4499	#ifdef VBOX_WITH_PGM_NEM_MODE
4500	const uint32_t cHostPages = RT_ALIGN_T(cb, HOST_PAGE_SIZE, RTGCPHYS) >> HOST_PAGE_SHIFT;
4501	#endif
4502
4503	/*
4504	* Make sure we've got a free ROM range.
4505	*/
4506	uint8_t const idRomRange = pVM->pgm.s.cRomRanges;
4507	AssertLogRelReturn(idRomRange < RT_ELEMENTS(pVM->pgm.s.apRomRanges), VERR_PGM_TOO_MANY_ROM_RANGES);
4508
4509	/*
4510	* Look thru the existing ROM range and make sure there aren't any
4511	* overlapping registration.
4512	*/
4513	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
4514	for (uint32_t idx = 0; idx < cRomRanges; idx++)
4515	{
4516	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
4517	AssertLogRelMsgReturn( GCPhys > pRom->GCPhysLast
4518	\|\| GCPhysLast < pRom->GCPhys,
4519	("%RGp-%RGp (%s) conflicts with existing %RGp-%RGp (%s)\n",
4520	GCPhys, GCPhysLast, pszDesc,
4521	pRom->GCPhys, pRom->GCPhysLast, pRom->pszDesc),
4522	VERR_PGM_RAM_CONFLICT);
4523	}
4524
4525	/*
4526	* Find the RAM location and check for conflicts.
4527	*
4528	* Conflict detection is a bit different than for RAM registration since a
4529	* ROM can be located within a RAM range. So, what we have to check for is
4530	* other memory types (other than RAM that is) and that we don't span more
4531	* than one RAM range (lazy).
4532	*/
4533	uint32_t idxInsert = UINT32_MAX;
4534	PPGMRAMRANGE const pOverlappingRange = pgmR3PhysRamRangeFindOverlapping(pVM, GCPhys, GCPhysLast, &idxInsert);
4535	if (pOverlappingRange)
4536	{
4537	/* completely within? */
4538	AssertLogRelMsgReturn( GCPhys >= pOverlappingRange->GCPhys
4539	&& GCPhysLast <= pOverlappingRange->GCPhysLast,
4540	("%RGp-%RGp (%s) falls partly outside %RGp-%RGp (%s)\n",
4541	GCPhys, GCPhysLast, pszDesc,
4542	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
4543	VERR_PGM_RAM_CONFLICT);
4544
4545	/* Check that is isn't an ad hoc range, but a real RAM range. */
4546	AssertLogRelMsgReturn(!PGM_RAM_RANGE_IS_AD_HOC(pOverlappingRange),
4547	("%RGp-%RGp (ROM/%s) mapping attempt in non-RAM range: %RGp-%RGp (%s)\n",
4548	GCPhys, GCPhysLast, pszDesc,
4549	pOverlappingRange->GCPhys, pOverlappingRange->GCPhysLast, pOverlappingRange->pszDesc),
4550	VERR_PGM_RAM_CONFLICT);
4551
4552	/* All the pages must be RAM pages. */
4553	PPGMPAGE pPage = &pOverlappingRange->aPages[(GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT];
4554	uint32_t cPagesLeft = cGuestPages;
4555	while (cPagesLeft-- > 0)
4556	{
4557	AssertLogRelMsgReturn(PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM,
4558	("%RGp (%R[pgmpage]) isn't a RAM page - registering %RGp-%RGp (%s).\n",
4559	GCPhys + ((RTGCPHYS)cPagesLeft << GUEST_PAGE_SHIFT), pPage, GCPhys, GCPhysLast, pszDesc),
4560	VERR_PGM_RAM_CONFLICT);
4561	AssertLogRelMsgReturn(PGM_PAGE_IS_ZERO(pPage) \|\| PGM_IS_IN_NEM_MODE(pVM),
4562	("%RGp (%R[pgmpage]) is not a ZERO page - registering %RGp-%RGp (%s).\n",
4563	GCPhys + ((RTGCPHYS)cPagesLeft << GUEST_PAGE_SHIFT), pPage, GCPhys, GCPhysLast, pszDesc),
4564	VERR_PGM_UNEXPECTED_PAGE_STATE);
4565	pPage++;
4566	}
4567	}
4568
4569	/*
4570	* Update the base memory reservation if necessary.
4571	*/
4572	uint32_t const cExtraBaseCost = (pOverlappingRange ? 0 : cGuestPages)
4573	+ (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? cGuestPages : 0);
4574	if (cExtraBaseCost)
4575	{
4576	int rc = MMR3IncreaseBaseReservation(pVM, cExtraBaseCost);
4577	AssertRCReturn(rc, rc);
4578	}
4579
4580	#ifdef VBOX_WITH_NATIVE_NEM
4581	/*
4582	* Early NEM notification before we've made any changes or anything.
4583	*/
4584	uint32_t const fNemNotify = (pOverlappingRange ? NEM_NOTIFY_PHYS_ROM_F_REPLACE : 0)
4585	\| (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED ? NEM_NOTIFY_PHYS_ROM_F_SHADOW : 0);
4586	uint8_t u2NemState = UINT8_MAX;
4587	uint32_t uNemRange = 0;
4588	if (VM_IS_NEM_ENABLED(pVM))
4589	{
4590	int rc = NEMR3NotifyPhysRomRegisterEarly(pVM, GCPhys, cGuestPages << GUEST_PAGE_SHIFT,
4591	pOverlappingRange
4592	? PGM_RAMRANGE_CALC_PAGE_R3PTR(pOverlappingRange, GCPhys) : NULL,
4593	fNemNotify, &u2NemState,
4594	pOverlappingRange ? &pOverlappingRange->uNemRange : &uNemRange);
4595	AssertLogRelRCReturn(rc, rc);
4596	}
4597	#endif
4598
4599	/*
4600	* Allocate memory for the virgin copy of the RAM. In simplified memory
4601	* mode, we allocate memory for any ad-hoc RAM range and for shadow pages.
4602	*/
4603	int rc;
4604	PGMMALLOCATEPAGESREQ pReq = NULL;
4605	#ifdef VBOX_WITH_PGM_NEM_MODE
4606	void *pvRam = NULL;
4607	void *pvAlt = NULL;
4608	if (PGM_IS_IN_NEM_MODE(pVM))
4609	{
4610	if (!pOverlappingRange)
4611	{
4612	rc = SUPR3PageAlloc(cHostPages, 0, &pvRam);
4613	if (RT_FAILURE(rc))
4614	return rc;
4615	}
4616	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4617	{
4618	rc = SUPR3PageAlloc(cHostPages, 0, &pvAlt);
4619	if (RT_FAILURE(rc))
4620	{
4621	if (pvRam)
4622	SUPR3PageFree(pvRam, cHostPages);
4623	return rc;
4624	}
4625	}
4626	}
4627	else
4628	#endif
4629	{
4630	rc = GMMR3AllocatePagesPrepare(pVM, &pReq, cGuestPages, GMMACCOUNT_BASE);
4631	AssertRCReturn(rc, rc);
4632
4633	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
4634	{
4635	pReq->aPages[iPage].HCPhysGCPhys = GCPhys + (iPage << GUEST_PAGE_SHIFT);
4636	pReq->aPages[iPage].fZeroed = false;
4637	pReq->aPages[iPage].idPage = NIL_GMM_PAGEID;
4638	pReq->aPages[iPage].idSharedPage = NIL_GMM_PAGEID;
4639	}
4640
4641	rc = GMMR3AllocatePagesPerform(pVM, pReq);
4642	if (RT_FAILURE(rc))
4643	{
4644	GMMR3AllocatePagesCleanup(pReq);
4645	return rc;
4646	}
4647	}
4648
4649	/*
4650	* Allocate a RAM range if required.
4651	* Note! We don't clean up the RAM range here on failure, VM destruction does that.
4652	*/
4653	rc = VINF_SUCCESS;
4654	PPGMRAMRANGE pRamRange = NULL;
4655	if (!pOverlappingRange)
4656	rc = pgmR3PhysAllocateRamRange(pVM, pVCpu, cGuestPages, PGM_RAM_RANGE_FLAGS_AD_HOC_ROM, &pRamRange);
4657	if (RT_SUCCESS(rc))
4658	{
4659	/*
4660	* Allocate a ROM range.
4661	* Note! We don't clean up the ROM range here on failure, VM destruction does that.
4662	*/
4663	if (SUPR3IsDriverless())
4664	rc = pgmPhysRomRangeAllocCommon(pVM, cGuestPages, idRomRange, fFlags);
4665	else
4666	{
4667	PGMPHYSROMALLOCATERANGEREQ RomRangeReq;
4668	RomRangeReq.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
4669	RomRangeReq.Hdr.cbReq = sizeof(RomRangeReq);
4670	RomRangeReq.cbGuestPage = GUEST_PAGE_SIZE;
4671	RomRangeReq.cGuestPages = cGuestPages;
4672	RomRangeReq.idRomRange = idRomRange;
4673	RomRangeReq.fFlags = fFlags;
4674	rc = VMMR3CallR0Emt(pVM, pVCpu, VMMR0_DO_PGM_PHYS_ROM_ALLOCATE_RANGE, 0 /u64Arg/, &RomRangeReq.Hdr);
4675	}
4676	}
4677	if (RT_SUCCESS(rc))
4678	{
4679	/*
4680	* Initialize and map the RAM range (if required).
4681	*/
4682	PPGMROMRANGE const pRomRange = pVM->pgm.s.apRomRanges[idRomRange];
4683	AssertPtr(pRomRange);
4684	uint32_t const idxFirstRamPage = pOverlappingRange ? (GCPhys - pOverlappingRange->GCPhys) >> GUEST_PAGE_SHIFT : 0;
4685	PPGMROMPAGE pRomPage = &pRomRange->aPages[0];
4686	if (!pOverlappingRange)
4687	{
4688	/* Initialize the new RAM range and insert it into the lookup table. */
4689	pRamRange->pszDesc = pszDesc;
4690	#ifdef VBOX_WITH_NATIVE_NEM
4691	pRamRange->uNemRange = uNemRange;
4692	#endif
4693
4694	PPGMPAGE pRamPage = &pRamRange->aPages[idxFirstRamPage];
4695	#ifdef VBOX_WITH_PGM_NEM_MODE
4696	if (PGM_IS_IN_NEM_MODE(pVM))
4697	{
4698	AssertPtr(pvRam); Assert(pReq == NULL);
4699	pRamRange->pbR3 = (uint8_t *)pvRam;
4700	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4701	{
4702	PGM_PAGE_INIT(pRamPage, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4703	PGMPAGETYPE_ROM, PGM_PAGE_STATE_ALLOCATED);
4704	pRomPage->Virgin = *pRamPage;
4705	}
4706	}
4707	else
4708	#endif
4709	{
4710	Assert(!pRamRange->pbR3); Assert(!pvRam);
4711	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4712	{
4713	PGM_PAGE_INIT(pRamPage,
4714	pReq->aPages[iPage].HCPhysGCPhys,
4715	pReq->aPages[iPage].idPage,
4716	PGMPAGETYPE_ROM,
4717	PGM_PAGE_STATE_ALLOCATED);
4718
4719	pRomPage->Virgin = *pRamPage;
4720	}
4721	}
4722
4723	pVM->pgm.s.cAllPages += cGuestPages;
4724	pVM->pgm.s.cPrivatePages += cGuestPages;
4725
4726	rc = pgmR3PhysRamRangeInsertLookup(pVM, pRamRange, GCPhys, &idxInsert);
4727	}
4728	else
4729	{
4730	/* Insert the ROM into an existing RAM range. */
4731	PPGMPAGE pRamPage = &pOverlappingRange->aPages[idxFirstRamPage];
4732	#ifdef VBOX_WITH_PGM_NEM_MODE
4733	if (PGM_IS_IN_NEM_MODE(pVM))
4734	{
4735	Assert(pvRam == NULL); Assert(pReq == NULL);
4736	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4737	{
4738	Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4739	Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4740	Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4741	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4742	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4743	PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4744	PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4745	PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4746
4747	pRomPage->Virgin = *pRamPage;
4748	}
4749	}
4750	else
4751	#endif
4752	{
4753	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4754	{
4755	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_ROM);
4756	PGM_PAGE_SET_HCPHYS(pVM, pRamPage, pReq->aPages[iPage].HCPhysGCPhys);
4757	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4758	PGM_PAGE_SET_PAGEID(pVM, pRamPage, pReq->aPages[iPage].idPage);
4759	PGM_PAGE_SET_PDE_TYPE(pVM, pRamPage, PGM_PAGE_PDE_TYPE_DONTCARE);
4760	PGM_PAGE_SET_PTE_INDEX(pVM, pRamPage, 0);
4761	PGM_PAGE_SET_TRACKING(pVM, pRamPage, 0);
4762
4763	pRomPage->Virgin = *pRamPage;
4764	}
4765	pVM->pgm.s.cZeroPages -= cGuestPages;
4766	pVM->pgm.s.cPrivatePages += cGuestPages;
4767	}
4768	pRamRange = pOverlappingRange;
4769	}
4770
4771	if (RT_SUCCESS(rc))
4772	{
4773	#ifdef VBOX_WITH_NATIVE_NEM
4774	/* Set the NEM state of the pages if needed. */
4775	if (u2NemState != UINT8_MAX)
4776	pgmPhysSetNemStateForPages(&pRamRange->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4777	#endif
4778
4779	/* Flush physical page map TLB. */
4780	pgmPhysInvalidatePageMapTLB(pVM);
4781
4782	/*
4783	* Register the ROM access handler.
4784	*/
4785	rc = PGMHandlerPhysicalRegister(pVM, GCPhys, GCPhysLast, pVM->pgm.s.hRomPhysHandlerType, idRomRange, pszDesc);
4786	if (RT_SUCCESS(rc))
4787	{
4788	/*
4789	* Copy the image over to the virgin pages.
4790	* This must be done after linking in the RAM range.
4791	*/
4792	size_t cbBinaryLeft = cbBinary;
4793	PPGMPAGE pRamPage = &pRamRange->aPages[idxFirstRamPage];
4794	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
4795	{
4796	void *pvDstPage;
4797	rc = pgmPhysPageMap(pVM, pRamPage, GCPhys + (iPage << GUEST_PAGE_SHIFT), &pvDstPage);
4798	if (RT_FAILURE(rc))
4799	{
4800	VMSetError(pVM, rc, RT_SRC_POS, "Failed to map virgin ROM page at %RGp", GCPhys);
4801	break;
4802	}
4803	if (cbBinaryLeft >= GUEST_PAGE_SIZE)
4804	{
4805	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), GUEST_PAGE_SIZE);
4806	cbBinaryLeft -= GUEST_PAGE_SIZE;
4807	}
4808	else
4809	{
4810	RT_BZERO(pvDstPage, GUEST_PAGE_SIZE); /* (shouldn't be necessary, but can't hurt either) */
4811	if (cbBinaryLeft > 0)
4812	{
4813	memcpy(pvDstPage, (uint8_t const *)pvBinary + ((size_t)iPage << GUEST_PAGE_SHIFT), cbBinaryLeft);
4814	cbBinaryLeft = 0;
4815	}
4816	}
4817	}
4818	if (RT_SUCCESS(rc))
4819	{
4820	/*
4821	* Initialize the ROM range.
4822	* Note that the Virgin member of the pages has already been initialized above.
4823	*/
4824	Assert(pRomRange->cb == cb);
4825	Assert(pRomRange->fFlags == fFlags);
4826	Assert(pRomRange->idSavedState == UINT8_MAX);
4827	pRomRange->GCPhys = GCPhys;
4828	pRomRange->GCPhysLast = GCPhysLast;
4829	pRomRange->cbOriginal = cbBinary;
4830	pRomRange->pszDesc = pszDesc;
4831	#ifdef VBOX_WITH_PGM_NEM_MODE
4832	pRomRange->pbR3Alternate = (uint8_t *)pvAlt;
4833	#endif
4834	pRomRange->pvOriginal = fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY
4835	? pvBinary : RTMemDup(pvBinary, cbBinary);
4836	if (pRomRange->pvOriginal)
4837	{
4838	for (unsigned iPage = 0; iPage < cGuestPages; iPage++)
4839	{
4840	PPGMROMPAGE const pPage = &pRomRange->aPages[iPage];
4841	pPage->enmProt = PGMROMPROT_READ_ROM_WRITE_IGNORE;
4842	#ifdef VBOX_WITH_PGM_NEM_MODE
4843	if (PGM_IS_IN_NEM_MODE(pVM))
4844	PGM_PAGE_INIT(&pPage->Shadow, UINT64_C(0x0000fffffffff000), NIL_GMM_PAGEID,
4845	PGMPAGETYPE_ROM_SHADOW, PGM_PAGE_STATE_ALLOCATED);
4846	else
4847	#endif
4848	PGM_PAGE_INIT_ZERO(&pPage->Shadow, pVM, PGMPAGETYPE_ROM_SHADOW);
4849	}
4850
4851	/* update the page count stats for the shadow pages. */
4852	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4853	{
4854	if (PGM_IS_IN_NEM_MODE(pVM))
4855	pVM->pgm.s.cPrivatePages += cGuestPages;
4856	else
4857	pVM->pgm.s.cZeroPages += cGuestPages;
4858	pVM->pgm.s.cAllPages += cGuestPages;
4859	}
4860
4861	#ifdef VBOX_WITH_NATIVE_NEM
4862	/*
4863	* Notify NEM again.
4864	*/
4865	if (VM_IS_NEM_ENABLED(pVM))
4866	{
4867	u2NemState = UINT8_MAX;
4868	rc = NEMR3NotifyPhysRomRegisterLate(pVM, GCPhys, cb, PGM_RAMRANGE_CALC_PAGE_R3PTR(pRamRange, GCPhys),
4869	fNemNotify, &u2NemState, &pRamRange->uNemRange);
4870	if (u2NemState != UINT8_MAX)
4871	pgmPhysSetNemStateForPages(&pRamRange->aPages[idxFirstRamPage], cGuestPages, u2NemState);
4872	}
4873	else
4874	#endif
4875	GMMR3AllocatePagesCleanup(pReq);
4876	if (RT_SUCCESS(rc))
4877	{
4878	/*
4879	* Done!
4880	*/
4881	#ifdef VBOX_STRICT
4882	pgmPhysAssertRamRangesLocked(pVM, false /fInUpdate/, false /fRamRelaxed/);
4883	#endif
4884	return rc;
4885	}
4886
4887	/*
4888	* bail out
4889	*/
4890	#ifdef VBOX_WITH_NATIVE_NEM
4891	if (fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
4892	{
4893	Assert(VM_IS_NEM_ENABLED(pVM));
4894	pVM->pgm.s.cPrivatePages -= cGuestPages;
4895	pVM->pgm.s.cAllPages -= cGuestPages;
4896	}
4897	#endif
4898	}
4899	else
4900	rc = VERR_NO_MEMORY;
4901	}
4902
4903	int rc2 = PGMHandlerPhysicalDeregister(pVM, GCPhys);
4904	AssertRC(rc2);
4905	}
4906
4907	idxInsert -= 1;
4908	if (!pOverlappingRange)
4909	pgmR3PhysRamRangeRemoveLookup(pVM, pRamRange, &idxInsert);
4910	}
4911	/* else: lookup insertion failed. */
4912
4913	if (pOverlappingRange)
4914	{
4915	PPGMPAGE pRamPage = &pOverlappingRange->aPages[idxFirstRamPage];
4916	#ifdef VBOX_WITH_PGM_NEM_MODE
4917	if (PGM_IS_IN_NEM_MODE(pVM))
4918	{
4919	Assert(pvRam == NULL); Assert(pReq == NULL);
4920	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++, pRomPage++)
4921	{
4922	Assert(PGM_PAGE_GET_HCPHYS(pRamPage) == UINT64_C(0x0000fffffffff000));
4923	Assert(PGM_PAGE_GET_PAGEID(pRamPage) == NIL_GMM_PAGEID);
4924	Assert(PGM_PAGE_GET_STATE(pRamPage) == PGM_PAGE_STATE_ALLOCATED);
4925	PGM_PAGE_SET_TYPE(pVM, pRamPage, PGMPAGETYPE_RAM);
4926	PGM_PAGE_SET_STATE(pVM, pRamPage, PGM_PAGE_STATE_ALLOCATED);
4927	}
4928	}
4929	else
4930	#endif
4931	{
4932	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++, pRamPage++)
4933	PGM_PAGE_INIT_ZERO(pRamPage, pVM, PGMPAGETYPE_RAM);
4934	pVM->pgm.s.cZeroPages += cGuestPages;
4935	pVM->pgm.s.cPrivatePages -= cGuestPages;
4936	}
4937	}
4938	}
4939	pgmPhysInvalidatePageMapTLB(pVM);
4940	pgmPhysInvalidRamRangeTlbs(pVM);
4941
4942	#ifdef VBOX_WITH_PGM_NEM_MODE
4943	if (PGM_IS_IN_NEM_MODE(pVM))
4944	{
4945	Assert(!pReq);
4946	if (pvRam)
4947	SUPR3PageFree(pvRam, cHostPages);
4948	if (pvAlt)
4949	SUPR3PageFree(pvAlt, cHostPages);
4950	}
4951	else
4952	#endif
4953	{
4954	GMMR3FreeAllocatedPages(pVM, pReq);
4955	GMMR3AllocatePagesCleanup(pReq);
4956	}
4957
4958	/* We don't bother to actually free either the ROM nor the RAM ranges
4959	themselves, as already mentioned above, we'll leave that to the VM
4960	termination cleanup code. */
4961	return rc;
4962	}
4963
4964
4965	/**
4966	* Registers a ROM image.
4967	*
4968	* Shadowed ROM images requires double the amount of backing memory, so,
4969	* don't use that unless you have to. Shadowing of ROM images is process
4970	* where we can select where the reads go and where the writes go. On real
4971	* hardware the chipset provides means to configure this. We provide
4972	* PGMR3PhysRomProtect() for this purpose.
4973	*
4974	* A read-only copy of the ROM image will always be kept around while we
4975	* will allocate RAM pages for the changes on demand (unless all memory
4976	* is configured to be preallocated).
4977	*
4978	* @returns VBox status code.
4979	* @param pVM The cross context VM structure.
4980	* @param pDevIns The device instance owning the ROM.
4981	* @param GCPhys First physical address in the range.
4982	* Must be page aligned!
4983	* @param cb The size of the range (in bytes).
4984	* Must be page aligned!
4985	* @param pvBinary Pointer to the binary data backing the ROM image.
4986	* @param cbBinary The size of the binary data pvBinary points to.
4987	* This must be less or equal to @a cb.
4988	* @param fFlags Mask of flags, PGMPHYS_ROM_FLAGS_XXX.
4989	* @param pszDesc Pointer to description string. This must not be freed.
4990	*
4991	* @remark There is no way to remove the rom, automatically on device cleanup or
4992	* manually from the device yet. This isn't difficult in any way, it's
4993	* just not something we expect to be necessary for a while.
4994	*/
4995	VMMR3DECL(int) PGMR3PhysRomRegister(PVM pVM, PPDMDEVINS pDevIns, RTGCPHYS GCPhys, RTGCPHYS cb,
4996	const void pvBinary, uint32_t cbBinary, uint8_t fFlags, const char pszDesc)
4997	{
4998	Log(("PGMR3PhysRomRegister: pDevIns=%p GCPhys=%RGp(-%RGp) cb=%RGp pvBinary=%p cbBinary=%#x fFlags=%#x pszDesc=%s\n",
4999	pDevIns, GCPhys, GCPhys + cb, cb, pvBinary, cbBinary, fFlags, pszDesc));
5000	PGM_LOCK_VOID(pVM);
5001
5002	int rc = pgmR3PhysRomRegisterLocked(pVM, pDevIns, GCPhys, cb, pvBinary, cbBinary, fFlags, pszDesc);
5003
5004	PGM_UNLOCK(pVM);
5005	return rc;
5006	}
5007
5008
5009	/**
5010	* Called by PGMR3MemSetup to reset the shadow, switch to the virgin, and verify
5011	* that the virgin part is untouched.
5012	*
5013	* This is done after the normal memory has been cleared.
5014	*
5015	* ASSUMES that the caller owns the PGM lock.
5016	*
5017	* @param pVM The cross context VM structure.
5018	*/
5019	int pgmR3PhysRomReset(PVM pVM)
5020	{
5021	PGM_LOCK_ASSERT_OWNER(pVM);
5022	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5023	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5024	{
5025	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5026	uint32_t const cGuestPages = pRom->cb >> GUEST_PAGE_SHIFT;
5027
5028	if (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED)
5029	{
5030	/*
5031	* Reset the physical handler.
5032	*/
5033	int rc = PGMR3PhysRomProtect(pVM, pRom->GCPhys, pRom->cb, PGMROMPROT_READ_ROM_WRITE_IGNORE);
5034	AssertRCReturn(rc, rc);
5035
5036	/*
5037	* What we do with the shadow pages depends on the memory
5038	* preallocation option. If not enabled, we'll just throw
5039	* out all the dirty pages and replace them by the zero page.
5040	*/
5041	#ifdef VBOX_WITH_PGM_NEM_MODE
5042	if (PGM_IS_IN_NEM_MODE(pVM))
5043	{
5044	/* Clear all the shadow pages (currently using alternate backing). */
5045	RT_BZERO(pRom->pbR3Alternate, pRom->cb);
5046	}
5047	else
5048	#endif
5049	if (!pVM->pgm.s.fRamPreAlloc)
5050	{
5051	/* Free the dirty pages. */
5052	uint32_t cPendingPages = 0;
5053	PGMMFREEPAGESREQ pReq;
5054	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5055	AssertRCReturn(rc, rc);
5056
5057	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
5058	if ( !PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow)
5059	&& !PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow))
5060	{
5061	Assert(PGM_PAGE_GET_STATE(&pRom->aPages[iPage].Shadow) == PGM_PAGE_STATE_ALLOCATED);
5062	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, &pRom->aPages[iPage].Shadow,
5063	pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT),
5064	(PGMPAGETYPE)PGM_PAGE_GET_TYPE(&pRom->aPages[iPage].Shadow));
5065	AssertLogRelRCReturn(rc, rc);
5066	}
5067
5068	if (cPendingPages)
5069	{
5070	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
5071	AssertLogRelRCReturn(rc, rc);
5072	}
5073	GMMR3FreePagesCleanup(pReq);
5074	}
5075	else
5076	{
5077	/* clear all the shadow pages. */
5078	for (uint32_t iPage = 0; iPage < cGuestPages; iPage++)
5079	{
5080	if (PGM_PAGE_IS_ZERO(&pRom->aPages[iPage].Shadow))
5081	continue;
5082	Assert(!PGM_PAGE_IS_BALLOONED(&pRom->aPages[iPage].Shadow));
5083	void *pvDstPage;
5084	RTGCPHYS const GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5085	rc = pgmPhysPageMakeWritableAndMap(pVM, &pRom->aPages[iPage].Shadow, GCPhys, &pvDstPage);
5086	if (RT_FAILURE(rc))
5087	break;
5088	RT_BZERO(pvDstPage, GUEST_PAGE_SIZE);
5089	}
5090	AssertRCReturn(rc, rc);
5091	}
5092	}
5093
5094	/*
5095	* Restore the original ROM pages after a saved state load.
5096	* Also, in strict builds check that ROM pages remain unmodified.
5097	*/
5098	#ifndef VBOX_STRICT
5099	if (pVM->pgm.s.fRestoreRomPagesOnReset)
5100	#endif
5101	{
5102	size_t cbSrcLeft = pRom->cbOriginal;
5103	uint8_t const pbSrcPage = (uint8_t const )pRom->pvOriginal;
5104	uint32_t cRestored = 0;
5105	for (uint32_t iPage = 0; iPage < cGuestPages && cbSrcLeft > 0; iPage++, pbSrcPage += GUEST_PAGE_SIZE)
5106	{
5107	RTGCPHYS const GCPhys = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5108	PPGMPAGE const pPage = pgmPhysGetPage(pVM, GCPhys);
5109	void const *pvDstPage = NULL;
5110	int rc = pgmPhysPageMapReadOnly(pVM, pPage, GCPhys, &pvDstPage);
5111	if (RT_FAILURE(rc))
5112	break;
5113
5114	if (memcmp(pvDstPage, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE)))
5115	{
5116	if (pVM->pgm.s.fRestoreRomPagesOnReset)
5117	{
5118	void *pvDstPageW = NULL;
5119	rc = pgmPhysPageMap(pVM, pPage, GCPhys, &pvDstPageW);
5120	AssertLogRelRCReturn(rc, rc);
5121	memcpy(pvDstPageW, pbSrcPage, RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE));
5122	cRestored++;
5123	}
5124	else
5125	LogRel(("pgmR3PhysRomReset: %RGp: ROM page changed (%s)\n", GCPhys, pRom->pszDesc));
5126	}
5127	cbSrcLeft -= RT_MIN(cbSrcLeft, GUEST_PAGE_SIZE);
5128	}
5129	if (cRestored > 0)
5130	LogRel(("PGM: ROM \"%s\": Reloaded %u of %u pages.\n", pRom->pszDesc, cRestored, cGuestPages));
5131	}
5132	}
5133
5134	/* Clear the ROM restore flag now as we only need to do this once after
5135	loading saved state. */
5136	pVM->pgm.s.fRestoreRomPagesOnReset = false;
5137
5138	return VINF_SUCCESS;
5139	}
5140
5141
5142	/**
5143	* Called by PGMR3Term to free resources.
5144	*
5145	* ASSUMES that the caller owns the PGM lock.
5146	*
5147	* @param pVM The cross context VM structure.
5148	*/
5149	void pgmR3PhysRomTerm(PVM pVM)
5150	{
5151	/*
5152	* Free the heap copy of the original bits.
5153	*/
5154	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5155	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5156	{
5157	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5158	if ( pRom->pvOriginal
5159	&& !(pRom->fFlags & PGMPHYS_ROM_FLAGS_PERMANENT_BINARY))
5160	{
5161	RTMemFree((void *)pRom->pvOriginal);
5162	pRom->pvOriginal = NULL;
5163	}
5164	}
5165	}
5166
5167
5168	/**
5169	* Change the shadowing of a range of ROM pages.
5170	*
5171	* This is intended for implementing chipset specific memory registers
5172	* and will not be very strict about the input. It will silently ignore
5173	* any pages that are not the part of a shadowed ROM.
5174	*
5175	* @returns VBox status code.
5176	* @retval VINF_PGM_SYNC_CR3
5177	*
5178	* @param pVM The cross context VM structure.
5179	* @param GCPhys Where to start. Page aligned.
5180	* @param cb How much to change. Page aligned.
5181	* @param enmProt The new ROM protection.
5182	*/
5183	VMMR3DECL(int) PGMR3PhysRomProtect(PVM pVM, RTGCPHYS GCPhys, RTGCPHYS cb, PGMROMPROT enmProt)
5184	{
5185	LogFlow(("PGMR3PhysRomProtect: GCPhys=%RGp cb=%RGp enmProt=%d\n", GCPhys, cb, enmProt));
5186
5187	/*
5188	* Check input
5189	*/
5190	if (!cb)
5191	return VINF_SUCCESS;
5192	AssertReturn(!(GCPhys & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
5193	AssertReturn(!(cb & GUEST_PAGE_OFFSET_MASK), VERR_INVALID_PARAMETER);
5194	RTGCPHYS GCPhysLast = GCPhys + (cb - 1);
5195	AssertReturn(GCPhysLast > GCPhys, VERR_INVALID_PARAMETER);
5196	AssertReturn(enmProt >= PGMROMPROT_INVALID && enmProt <= PGMROMPROT_END, VERR_INVALID_PARAMETER);
5197
5198	/*
5199	* Process the request.
5200	*/
5201	PGM_LOCK_VOID(pVM);
5202	int rc = VINF_SUCCESS;
5203	bool fFlushTLB = false;
5204	uint32_t const cRomRanges = RT_MIN(pVM->pgm.s.cRomRanges, RT_ELEMENTS(pVM->pgm.s.apRomRanges));
5205	for (uint32_t idx = 0; idx < cRomRanges; idx++)
5206	{
5207	PPGMROMRANGE const pRom = pVM->pgm.s.apRomRanges[idx];
5208	if ( GCPhys <= pRom->GCPhysLast
5209	&& GCPhysLast >= pRom->GCPhys
5210	&& (pRom->fFlags & PGMPHYS_ROM_FLAGS_SHADOWED))
5211	{
5212	/*
5213	* Iterate the relevant pages and make necessary the changes.
5214	*/
5215	#ifdef VBOX_WITH_NATIVE_NEM
5216	PPGMRAMRANGE const pRam = pgmPhysGetRange(pVM, GCPhys);
5217	AssertPtrReturn(pRam, VERR_INTERNAL_ERROR_3);
5218	#endif
5219	bool fChanges = false;
5220	uint32_t const cPages = pRom->GCPhysLast <= GCPhysLast
5221	? pRom->cb >> GUEST_PAGE_SHIFT
5222	: (GCPhysLast - pRom->GCPhys + 1) >> GUEST_PAGE_SHIFT;
5223	for (uint32_t iPage = (GCPhys - pRom->GCPhys) >> GUEST_PAGE_SHIFT;
5224	iPage < cPages;
5225	iPage++)
5226	{
5227	PPGMROMPAGE pRomPage = &pRom->aPages[iPage];
5228	if (PGMROMPROT_IS_ROM(pRomPage->enmProt) != PGMROMPROT_IS_ROM(enmProt))
5229	{
5230	fChanges = true;
5231
5232	/* flush references to the page. */
5233	RTGCPHYS const GCPhysPage = pRom->GCPhys + (iPage << GUEST_PAGE_SHIFT);
5234	PPGMPAGE pRamPage = pgmPhysGetPage(pVM, GCPhysPage);
5235	int rc2 = pgmPoolTrackUpdateGCPhys(pVM, GCPhysPage, pRamPage, true /fFlushPTEs/, &fFlushTLB);
5236	if (rc2 != VINF_SUCCESS && (rc == VINF_SUCCESS \|\| RT_FAILURE(rc2)))
5237	rc = rc2;
5238	#ifdef VBOX_WITH_NATIVE_NEM
5239	uint8_t u2State = PGM_PAGE_GET_NEM_STATE(pRamPage);
5240	#endif
5241
5242	PPGMPAGE pOld = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Virgin : &pRomPage->Shadow;
5243	PPGMPAGE pNew = PGMROMPROT_IS_ROM(pRomPage->enmProt) ? &pRomPage->Shadow : &pRomPage->Virgin;
5244
5245	pOld = pRamPage;
5246	pRamPage = pNew;
5247	/** @todo preserve the volatile flags (handlers) when these have been moved out of HCPhys! */
5248
5249	#ifdef VBOX_WITH_NATIVE_NEM
5250	# ifdef VBOX_WITH_PGM_NEM_MODE
5251	/* In simplified mode we have to switch the page data around too. */
5252	if (PGM_IS_IN_NEM_MODE(pVM))
5253	{
5254	uint8_t abPage[GUEST_PAGE_SIZE];
5255	uint8_t * const pbRamPage = PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage);
5256	memcpy(abPage, &pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], sizeof(abPage));
5257	memcpy(&pRom->pbR3Alternate[(size_t)iPage << GUEST_PAGE_SHIFT], pbRamPage, sizeof(abPage));
5258	memcpy(pbRamPage, abPage, sizeof(abPage));
5259	}
5260	# endif
5261	/* Tell NEM about the backing and protection change. */
5262	if (VM_IS_NEM_ENABLED(pVM))
5263	{
5264	PGMPAGETYPE enmType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pNew);
5265	NEMHCNotifyPhysPageChanged(pVM, GCPhys, PGM_PAGE_GET_HCPHYS(pOld), PGM_PAGE_GET_HCPHYS(pNew),
5266	PGM_RAMRANGE_CALC_PAGE_R3PTR(pRam, GCPhysPage),
5267	pgmPhysPageCalcNemProtection(pRamPage, enmType), enmType, &u2State);
5268	PGM_PAGE_SET_NEM_STATE(pRamPage, u2State);
5269	}
5270	#endif
5271	}
5272	pRomPage->enmProt = enmProt;
5273	}
5274
5275	/*
5276	* Reset the access handler if we made changes, no need to optimize this.
5277	*/
5278	if (fChanges)
5279	{
5280	int rc2 = PGMHandlerPhysicalReset(pVM, pRom->GCPhys);
5281	if (RT_FAILURE(rc2))
5282	{
5283	PGM_UNLOCK(pVM);
5284	AssertRC(rc);
5285	return rc2;
5286	}
5287
5288	/* Explicitly flush IEM. Not sure if this is really necessary, but better
5289	be on the safe side. This shouldn't be a high volume flush source. */
5290	IEMTlbInvalidateAllPhysicalAllCpus(pVM, NIL_VMCPUID, IEMTLBPHYSFLUSHREASON_ROM_PROTECT);
5291	}
5292
5293	/* Advance - cb isn't updated. */
5294	GCPhys = pRom->GCPhys + (cPages << GUEST_PAGE_SHIFT);
5295	}
5296	}
5297	PGM_UNLOCK(pVM);
5298	if (fFlushTLB)
5299	PGM_INVL_ALL_VCPU_TLBS(pVM);
5300
5301	return rc;
5302	}
5303
5304
5305
5306	/*********************************************************************************************************************************
5307	* Ballooning *
5308	*********************************************************************************************************************************/
5309
5310	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
5311
5312	/**
5313	* Rendezvous callback used by PGMR3ChangeMemBalloon that changes the memory balloon size
5314	*
5315	* This is only called on one of the EMTs while the other ones are waiting for
5316	* it to complete this function.
5317	*
5318	* @returns VINF_SUCCESS (VBox strict status code).
5319	* @param pVM The cross context VM structure.
5320	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5321	* @param pvUser User parameter
5322	*/
5323	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysChangeMemBalloonRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5324	{
5325	uintptr_t paUser = (uintptr_t )pvUser;
5326	bool fInflate = !!paUser[0];
5327	unsigned cPages = paUser[1];
5328	RTGCPHYS paPhysPage = (RTGCPHYS )paUser[2];
5329	uint32_t cPendingPages = 0;
5330	PGMMFREEPAGESREQ pReq;
5331	int rc;
5332
5333	Log(("pgmR3PhysChangeMemBalloonRendezvous: %s %x pages\n", (fInflate) ? "inflate" : "deflate", cPages));
5334	PGM_LOCK_VOID(pVM);
5335
5336	if (fInflate)
5337	{
5338	/* Flush the PGM pool cache as we might have stale references to pages that we just freed. */
5339	pgmR3PoolClearAllRendezvous(pVM, pVCpu, NULL);
5340
5341	/* Replace pages with ZERO pages. */
5342	rc = GMMR3FreePagesPrepare(pVM, &pReq, PGMPHYS_FREE_PAGE_BATCH_SIZE, GMMACCOUNT_BASE);
5343	if (RT_FAILURE(rc))
5344	{
5345	PGM_UNLOCK(pVM);
5346	AssertLogRelRC(rc);
5347	return rc;
5348	}
5349
5350	/* Iterate the pages. */
5351	for (unsigned i = 0; i < cPages; i++)
5352	{
5353	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5354	if ( pPage == NULL
5355	\|\| PGM_PAGE_GET_TYPE(pPage) != PGMPAGETYPE_RAM)
5356	{
5357	Log(("pgmR3PhysChangeMemBalloonRendezvous: invalid physical page %RGp pPage->u3Type=%d\n", paPhysPage[i], pPage ? PGM_PAGE_GET_TYPE(pPage) : 0));
5358	break;
5359	}
5360
5361	LogFlow(("balloon page: %RGp\n", paPhysPage[i]));
5362
5363	/* Flush the shadow PT if this page was previously used as a guest page table. */
5364	pgmPoolFlushPageByGCPhys(pVM, paPhysPage[i]);
5365
5366	rc = pgmPhysFreePage(pVM, pReq, &cPendingPages, pPage, paPhysPage[i], (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage));
5367	if (RT_FAILURE(rc))
5368	{
5369	PGM_UNLOCK(pVM);
5370	AssertLogRelRC(rc);
5371	return rc;
5372	}
5373	Assert(PGM_PAGE_IS_ZERO(pPage));
5374	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_BALLOONED);
5375	}
5376
5377	if (cPendingPages)
5378	{
5379	rc = GMMR3FreePagesPerform(pVM, pReq, cPendingPages);
5380	if (RT_FAILURE(rc))
5381	{
5382	PGM_UNLOCK(pVM);
5383	AssertLogRelRC(rc);
5384	return rc;
5385	}
5386	}
5387	GMMR3FreePagesCleanup(pReq);
5388	}
5389	else
5390	{
5391	/* Iterate the pages. */
5392	for (unsigned i = 0; i < cPages; i++)
5393	{
5394	PPGMPAGE pPage = pgmPhysGetPage(pVM, paPhysPage[i]);
5395	AssertBreak(pPage && PGM_PAGE_GET_TYPE(pPage) == PGMPAGETYPE_RAM);
5396
5397	LogFlow(("Free ballooned page: %RGp\n", paPhysPage[i]));
5398
5399	Assert(PGM_PAGE_IS_BALLOONED(pPage));
5400
5401	/* Change back to zero page. (NEM does not need to be informed.) */
5402	PGM_PAGE_SET_STATE(pVM, pPage, PGM_PAGE_STATE_ZERO);
5403	}
5404
5405	/* Note that we currently do not map any ballooned pages in our shadow page tables, so no need to flush the pgm pool. */
5406	}
5407
5408	/* Notify GMM about the balloon change. */
5409	rc = GMMR3BalloonedPages(pVM, (fInflate) ? GMMBALLOONACTION_INFLATE : GMMBALLOONACTION_DEFLATE, cPages);
5410	if (RT_SUCCESS(rc))
5411	{
5412	if (!fInflate)
5413	{
5414	Assert(pVM->pgm.s.cBalloonedPages >= cPages);
5415	pVM->pgm.s.cBalloonedPages -= cPages;
5416	}
5417	else
5418	pVM->pgm.s.cBalloonedPages += cPages;
5419	}
5420
5421	PGM_UNLOCK(pVM);
5422
5423	/* Flush the recompiler's TLB as well. */
5424	for (VMCPUID i = 0; i < pVM->cCpus; i++)
5425	CPUMSetChangedFlags(pVM->apCpusR3[i], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5426
5427	AssertLogRelRC(rc);
5428	return rc;
5429	}
5430
5431
5432	/**
5433	* Frees a range of ram pages, replacing them with ZERO pages; helper for PGMR3PhysFreeRamPages
5434	*
5435	* @param pVM The cross context VM structure.
5436	* @param fInflate Inflate or deflate memory balloon
5437	* @param cPages Number of pages to free
5438	* @param paPhysPage Array of guest physical addresses
5439	*/
5440	static DECLCALLBACK(void) pgmR3PhysChangeMemBalloonHelper(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5441	{
5442	uintptr_t paUser[3];
5443
5444	paUser[0] = fInflate;
5445	paUser[1] = cPages;
5446	paUser[2] = (uintptr_t)paPhysPage;
5447	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5448	AssertRC(rc);
5449
5450	/* Made a copy in PGMR3PhysFreeRamPages; free it here. */
5451	RTMemFree(paPhysPage);
5452	}
5453
5454	#endif /* 64-bit host && (Windows \|\| Solaris \|\| Linux \|\| FreeBSD) */
5455
5456	/**
5457	* Inflate or deflate a memory balloon
5458	*
5459	* @returns VBox status code.
5460	* @param pVM The cross context VM structure.
5461	* @param fInflate Inflate or deflate memory balloon
5462	* @param cPages Number of pages to free
5463	* @param paPhysPage Array of guest physical addresses
5464	*/
5465	VMMR3DECL(int) PGMR3PhysChangeMemBalloon(PVM pVM, bool fInflate, unsigned cPages, RTGCPHYS *paPhysPage)
5466	{
5467	/* This must match GMMR0Init; currently we only support memory ballooning on all 64-bit hosts except Mac OS X */
5468	#if HC_ARCH_BITS == 64 && (defined(RT_OS_WINDOWS) \|\| defined(RT_OS_SOLARIS) \|\| defined(RT_OS_LINUX) \|\| defined(RT_OS_FREEBSD))
5469	int rc;
5470
5471	/* Older additions (ancient non-functioning balloon code) pass wrong physical addresses. */
5472	AssertReturn(!(paPhysPage[0] & 0xfff), VERR_INVALID_PARAMETER);
5473
5474	/* We own the IOM lock here and could cause a deadlock by waiting for another VCPU that is blocking on the IOM lock.
5475	* In the SMP case we post a request packet to postpone the job.
5476	*/
5477	if (pVM->cCpus > 1)
5478	{
5479	unsigned cbPhysPage = cPages * sizeof(paPhysPage[0]);
5480	RTGCPHYS paPhysPageCopy = (RTGCPHYS )RTMemAlloc(cbPhysPage);
5481	AssertReturn(paPhysPageCopy, VERR_NO_MEMORY);
5482
5483	memcpy(paPhysPageCopy, paPhysPage, cbPhysPage);
5484
5485	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysChangeMemBalloonHelper, 4, pVM, fInflate, cPages, paPhysPageCopy);
5486	AssertRC(rc);
5487	}
5488	else
5489	{
5490	uintptr_t paUser[3];
5491
5492	paUser[0] = fInflate;
5493	paUser[1] = cPages;
5494	paUser[2] = (uintptr_t)paPhysPage;
5495	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysChangeMemBalloonRendezvous, (void *)paUser);
5496	AssertRC(rc);
5497	}
5498	return rc;
5499
5500	#else
5501	NOREF(pVM); NOREF(fInflate); NOREF(cPages); NOREF(paPhysPage);
5502	return VERR_NOT_IMPLEMENTED;
5503	#endif
5504	}
5505
5506
5507
5508	/*********************************************************************************************************************************
5509	* Write Monitoring *
5510	*********************************************************************************************************************************/
5511
5512	/**
5513	* Rendezvous callback used by PGMR3WriteProtectRAM that write protects all
5514	* physical RAM.
5515	*
5516	* This is only called on one of the EMTs while the other ones are waiting for
5517	* it to complete this function.
5518	*
5519	* @returns VINF_SUCCESS (VBox strict status code).
5520	* @param pVM The cross context VM structure.
5521	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5522	* @param pvUser User parameter, unused.
5523	*/
5524	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysWriteProtectRAMRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5525	{
5526	int rc = VINF_SUCCESS;
5527	NOREF(pvUser); NOREF(pVCpu);
5528
5529	PGM_LOCK_VOID(pVM);
5530	#ifdef PGMPOOL_WITH_OPTIMIZED_DIRTY_PT
5531	pgmPoolResetDirtyPages(pVM);
5532	#endif
5533
5534	uint32_t const cLookupEntries = RT_MIN(pVM->pgm.s.RamRangeUnion.cLookupEntries, RT_ELEMENTS(pVM->pgm.s.aRamRangeLookup));
5535	for (uint32_t idxLookup = 0; idxLookup < cLookupEntries; idxLookup++)
5536	{
5537	uint32_t const idRamRange = PGMRAMRANGELOOKUPENTRY_GET_ID(pVM->pgm.s.aRamRangeLookup[idxLookup]);
5538	AssertContinue(idRamRange < RT_ELEMENTS(pVM->pgm.s.apRamRanges));
5539	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
5540	AssertContinue(pRam);
5541
5542	uint32_t cPages = pRam->cb >> GUEST_PAGE_SHIFT;
5543	for (uint32_t iPage = 0; iPage < cPages; iPage++)
5544	{
5545	PPGMPAGE const pPage = &pRam->aPages[iPage];
5546	PGMPAGETYPE const enmPageType = (PGMPAGETYPE)PGM_PAGE_GET_TYPE(pPage);
5547
5548	if ( RT_LIKELY(enmPageType == PGMPAGETYPE_RAM)
5549	\|\| enmPageType == PGMPAGETYPE_MMIO2)
5550	{
5551	/*
5552	* A RAM page.
5553	*/
5554	switch (PGM_PAGE_GET_STATE(pPage))
5555	{
5556	case PGM_PAGE_STATE_ALLOCATED:
5557	/** @todo Optimize this: Don't always re-enable write
5558	* monitoring if the page is known to be very busy. */
5559	if (PGM_PAGE_IS_WRITTEN_TO(pPage))
5560	PGM_PAGE_CLEAR_WRITTEN_TO(pVM, pPage);
5561
5562	pgmPhysPageWriteMonitor(pVM, pPage, pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT));
5563	break;
5564
5565	case PGM_PAGE_STATE_SHARED:
5566	AssertFailed();
5567	break;
5568
5569	case PGM_PAGE_STATE_WRITE_MONITORED: /* nothing to change. */
5570	default:
5571	break;
5572	}
5573	}
5574	}
5575	}
5576	pgmR3PoolWriteProtectPages(pVM);
5577	PGM_INVL_ALL_VCPU_TLBS(pVM);
5578	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5579	CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5580
5581	PGM_UNLOCK(pVM);
5582	return rc;
5583	}
5584
5585	/**
5586	* Protect all physical RAM to monitor writes
5587	*
5588	* @returns VBox status code.
5589	* @param pVM The cross context VM structure.
5590	*/
5591	VMMR3DECL(int) PGMR3PhysWriteProtectRAM(PVM pVM)
5592	{
5593	VM_ASSERT_EMT_RETURN(pVM, VERR_VM_THREAD_NOT_EMT);
5594
5595	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysWriteProtectRAMRendezvous, NULL);
5596	AssertRC(rc);
5597	return rc;
5598	}
5599
5600
5601	/*********************************************************************************************************************************
5602	* Stats. *
5603	*********************************************************************************************************************************/
5604
5605	/**
5606	* Query the amount of free memory inside VMMR0
5607	*
5608	* @returns VBox status code.
5609	* @param pUVM The user mode VM handle.
5610	* @param pcbAllocMem Where to return the amount of memory allocated
5611	* by VMs.
5612	* @param pcbFreeMem Where to return the amount of memory that is
5613	* allocated from the host but not currently used
5614	* by any VMs.
5615	* @param pcbBallonedMem Where to return the sum of memory that is
5616	* currently ballooned by the VMs.
5617	* @param pcbSharedMem Where to return the amount of memory that is
5618	* currently shared.
5619	*/
5620	VMMR3DECL(int) PGMR3QueryGlobalMemoryStats(PUVM pUVM, uint64_t pcbAllocMem, uint64_t pcbFreeMem,
5621	uint64_t pcbBallonedMem, uint64_t pcbSharedMem)
5622	{
5623	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5624	VM_ASSERT_VALID_EXT_RETURN(pUVM->pVM, VERR_INVALID_VM_HANDLE);
5625
5626	uint64_t cAllocPages = 0;
5627	uint64_t cFreePages = 0;
5628	uint64_t cBalloonPages = 0;
5629	uint64_t cSharedPages = 0;
5630	if (!SUPR3IsDriverless())
5631	{
5632	int rc = GMMR3QueryHypervisorMemoryStats(pUVM->pVM, &cAllocPages, &cFreePages, &cBalloonPages, &cSharedPages);
5633	AssertRCReturn(rc, rc);
5634	}
5635
5636	if (pcbAllocMem)
5637	pcbAllocMem = cAllocPages _4K;
5638
5639	if (pcbFreeMem)
5640	pcbFreeMem = cFreePages _4K;
5641
5642	if (pcbBallonedMem)
5643	pcbBallonedMem = cBalloonPages _4K;
5644
5645	if (pcbSharedMem)
5646	pcbSharedMem = cSharedPages _4K;
5647
5648	Log(("PGMR3QueryVMMMemoryStats: all=%llx free=%llx ballooned=%llx shared=%llx\n",
5649	cAllocPages, cFreePages, cBalloonPages, cSharedPages));
5650	return VINF_SUCCESS;
5651	}
5652
5653
5654	/**
5655	* Query memory stats for the VM.
5656	*
5657	* @returns VBox status code.
5658	* @param pUVM The user mode VM handle.
5659	* @param pcbTotalMem Where to return total amount memory the VM may
5660	* possibly use.
5661	* @param pcbPrivateMem Where to return the amount of private memory
5662	* currently allocated.
5663	* @param pcbSharedMem Where to return the amount of actually shared
5664	* memory currently used by the VM.
5665	* @param pcbZeroMem Where to return the amount of memory backed by
5666	* zero pages.
5667	*
5668	* @remarks The total mem is normally larger than the sum of the three
5669	* components. There are two reasons for this, first the amount of
5670	* shared memory is what we're sure is shared instead of what could
5671	* possibly be shared with someone. Secondly, because the total may
5672	* include some pure MMIO pages that doesn't go into any of the three
5673	* sub-counts.
5674	*
5675	* @todo Why do we return reused shared pages instead of anything that could
5676	* potentially be shared? Doesn't this mean the first VM gets a much
5677	* lower number of shared pages?
5678	*/
5679	VMMR3DECL(int) PGMR3QueryMemoryStats(PUVM pUVM, uint64_t pcbTotalMem, uint64_t pcbPrivateMem,
5680	uint64_t pcbSharedMem, uint64_t pcbZeroMem)
5681	{
5682	UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
5683	PVM pVM = pUVM->pVM;
5684	VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
5685
5686	if (pcbTotalMem)
5687	pcbTotalMem = (uint64_t)pVM->pgm.s.cAllPages GUEST_PAGE_SIZE;
5688
5689	if (pcbPrivateMem)
5690	pcbPrivateMem = (uint64_t)pVM->pgm.s.cPrivatePages GUEST_PAGE_SIZE;
5691
5692	if (pcbSharedMem)
5693	pcbSharedMem = (uint64_t)pVM->pgm.s.cReusedSharedPages GUEST_PAGE_SIZE;
5694
5695	if (pcbZeroMem)
5696	pcbZeroMem = (uint64_t)pVM->pgm.s.cZeroPages GUEST_PAGE_SIZE;
5697
5698	Log(("PGMR3QueryMemoryStats: all=%x private=%x reused=%x zero=%x\n", pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cReusedSharedPages, pVM->pgm.s.cZeroPages));
5699	return VINF_SUCCESS;
5700	}
5701
5702
5703
5704	/*********************************************************************************************************************************
5705	* Chunk Mappings and Page Allocation *
5706	*********************************************************************************************************************************/
5707
5708	/**
5709	* Tree enumeration callback for dealing with age rollover.
5710	* It will perform a simple compression of the current age.
5711	*/
5712	static DECLCALLBACK(int) pgmR3PhysChunkAgeingRolloverCallback(PAVLU32NODECORE pNode, void *pvUser)
5713	{
5714	/* Age compression - ASSUMES iNow == 4. */
5715	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5716	if (pChunk->iLastUsed >= UINT32_C(0xffffff00))
5717	pChunk->iLastUsed = 3;
5718	else if (pChunk->iLastUsed >= UINT32_C(0xfffff000))
5719	pChunk->iLastUsed = 2;
5720	else if (pChunk->iLastUsed)
5721	pChunk->iLastUsed = 1;
5722	else /* iLastUsed = 0 */
5723	pChunk->iLastUsed = 4;
5724
5725	NOREF(pvUser);
5726	return 0;
5727	}
5728
5729
5730	/**
5731	* The structure passed in the pvUser argument of pgmR3PhysChunkUnmapCandidateCallback().
5732	*/
5733	typedef struct PGMR3PHYSCHUNKUNMAPCB
5734	{
5735	PVM pVM; /*< Pointer to the VM. /
5736	PPGMCHUNKR3MAP pChunk; /*< The chunk to unmap. /
5737	} PGMR3PHYSCHUNKUNMAPCB, *PPGMR3PHYSCHUNKUNMAPCB;
5738
5739
5740	/**
5741	* Callback used to find the mapping that's been unused for
5742	* the longest time.
5743	*/
5744	static DECLCALLBACK(int) pgmR3PhysChunkUnmapCandidateCallback(PAVLU32NODECORE pNode, void *pvUser)
5745	{
5746	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)pNode;
5747	PPGMR3PHYSCHUNKUNMAPCB pArg = (PPGMR3PHYSCHUNKUNMAPCB)pvUser;
5748
5749	/*
5750	* Check for locks and compare when last used.
5751	*/
5752	if (pChunk->cRefs)
5753	return 0;
5754	if (pChunk->cPermRefs)
5755	return 0;
5756	if ( pArg->pChunk
5757	&& pChunk->iLastUsed >= pArg->pChunk->iLastUsed)
5758	return 0;
5759
5760	/*
5761	* Check that it's not in any of the TLBs.
5762	*/
5763	PVM pVM = pArg->pVM;
5764	if ( pVM->pgm.s.ChunkR3Map.Tlb.aEntries[PGM_CHUNKR3MAPTLB_IDX(pChunk->Core.Key)].idChunk
5765	== pChunk->Core.Key)
5766	{
5767	pChunk = NULL;
5768	return 0;
5769	}
5770	#ifdef VBOX_STRICT
5771	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
5772	{
5773	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk != pChunk);
5774	Assert(pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk != pChunk->Core.Key);
5775	}
5776	#endif
5777
5778	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.PhysTlbR3.aEntries); i++)
5779	if (pVM->pgm.s.PhysTlbR3.aEntries[i].pMap == pChunk)
5780	return 0;
5781
5782	pArg->pChunk = pChunk;
5783	return 0;
5784	}
5785
5786
5787	/**
5788	* Finds a good candidate for unmapping when the ring-3 mapping cache is full.
5789	*
5790	* The candidate will not be part of any TLBs, so no need to flush
5791	* anything afterwards.
5792	*
5793	* @returns Chunk id.
5794	* @param pVM The cross context VM structure.
5795	*/
5796	static int32_t pgmR3PhysChunkFindUnmapCandidate(PVM pVM)
5797	{
5798	PGM_LOCK_ASSERT_OWNER(pVM);
5799
5800	/*
5801	* Enumerate the age tree starting with the left most node.
5802	*/
5803	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5804	PGMR3PHYSCHUNKUNMAPCB Args;
5805	Args.pVM = pVM;
5806	Args.pChunk = NULL;
5807	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkUnmapCandidateCallback, &Args);
5808	Assert(Args.pChunk);
5809	if (Args.pChunk)
5810	{
5811	Assert(Args.pChunk->cRefs == 0);
5812	Assert(Args.pChunk->cPermRefs == 0);
5813	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5814	return Args.pChunk->Core.Key;
5815	}
5816
5817	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkFindCandidate, a);
5818	return INT32_MAX;
5819	}
5820
5821
5822	/**
5823	* Rendezvous callback used by pgmR3PhysUnmapChunk that unmaps a chunk
5824	*
5825	* This is only called on one of the EMTs while the other ones are waiting for
5826	* it to complete this function.
5827	*
5828	* @returns VINF_SUCCESS (VBox strict status code).
5829	* @param pVM The cross context VM structure.
5830	* @param pVCpu The cross context virtual CPU structure of the calling EMT. Unused.
5831	* @param pvUser User pointer. Unused
5832	*
5833	*/
5834	static DECLCALLBACK(VBOXSTRICTRC) pgmR3PhysUnmapChunkRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
5835	{
5836	int rc = VINF_SUCCESS;
5837	PGM_LOCK_VOID(pVM);
5838	NOREF(pVCpu); NOREF(pvUser);
5839
5840	if (pVM->pgm.s.ChunkR3Map.c >= pVM->pgm.s.ChunkR3Map.cMax)
5841	{
5842	/* Flush the pgm pool cache; call the internal rendezvous handler as we're already in a rendezvous handler here. */
5843	/** @todo also not really efficient to unmap a chunk that contains PD
5844	* or PT pages. */
5845	pgmR3PoolClearAllRendezvous(pVM, pVM->apCpusR3[0], NULL /* no need to flush the REM TLB as we already did that above */);
5846
5847	/*
5848	* Request the ring-0 part to unmap a chunk to make space in the mapping cache.
5849	*/
5850	GMMMAPUNMAPCHUNKREQ Req;
5851	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5852	Req.Hdr.cbReq = sizeof(Req);
5853	Req.pvR3 = NULL;
5854	Req.idChunkMap = NIL_GMM_CHUNKID;
5855	Req.idChunkUnmap = pgmR3PhysChunkFindUnmapCandidate(pVM);
5856	if (Req.idChunkUnmap != INT32_MAX)
5857	{
5858	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5859	rc = VMMR3CallR0(pVM, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5860	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkUnmap, a);
5861	if (RT_SUCCESS(rc))
5862	{
5863	/*
5864	* Remove the unmapped one.
5865	*/
5866	PPGMCHUNKR3MAP pUnmappedChunk = (PPGMCHUNKR3MAP)RTAvlU32Remove(&pVM->pgm.s.ChunkR3Map.pTree, Req.idChunkUnmap);
5867	AssertRelease(pUnmappedChunk);
5868	AssertRelease(!pUnmappedChunk->cRefs);
5869	AssertRelease(!pUnmappedChunk->cPermRefs);
5870	pUnmappedChunk->pv = NULL;
5871	pUnmappedChunk->Core.Key = UINT32_MAX;
5872	MMR3HeapFree(pUnmappedChunk);
5873	pVM->pgm.s.ChunkR3Map.c--;
5874	pVM->pgm.s.cUnmappedChunks++;
5875
5876	/*
5877	* Flush dangling PGM pointers (R3 & R0 ptrs to GC physical addresses).
5878	*/
5879	/** @todo We should not flush chunks which include cr3 mappings. */
5880	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
5881	{
5882	PPGMCPU pPGM = &pVM->apCpusR3[idCpu]->pgm.s;
5883
5884	pPGM->pGst32BitPdR3 = NULL;
5885	pPGM->pGstPaePdptR3 = NULL;
5886	pPGM->pGstAmd64Pml4R3 = NULL;
5887	pPGM->pGstEptPml4R3 = NULL;
5888	pPGM->pGst32BitPdR0 = NIL_RTR0PTR;
5889	pPGM->pGstPaePdptR0 = NIL_RTR0PTR;
5890	pPGM->pGstAmd64Pml4R0 = NIL_RTR0PTR;
5891	pPGM->pGstEptPml4R0 = NIL_RTR0PTR;
5892	for (unsigned i = 0; i < RT_ELEMENTS(pPGM->apGstPaePDsR3); i++)
5893	{
5894	pPGM->apGstPaePDsR3[i] = NULL;
5895	pPGM->apGstPaePDsR0[i] = NIL_RTR0PTR;
5896	}
5897
5898	/* Flush REM TLBs. */
5899	CPUMSetChangedFlags(pVM->apCpusR3[idCpu], CPUM_CHANGED_GLOBAL_TLB_FLUSH);
5900	}
5901	}
5902	}
5903	}
5904	PGM_UNLOCK(pVM);
5905	return rc;
5906	}
5907
5908	/**
5909	* Unmap a chunk to free up virtual address space (request packet handler for pgmR3PhysChunkMap)
5910	*
5911	* @param pVM The cross context VM structure.
5912	*/
5913	static DECLCALLBACK(void) pgmR3PhysUnmapChunk(PVM pVM)
5914	{
5915	int rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, pgmR3PhysUnmapChunkRendezvous, NULL);
5916	AssertRC(rc);
5917	}
5918
5919
5920	/**
5921	* Maps the given chunk into the ring-3 mapping cache.
5922	*
5923	* This will call ring-0.
5924	*
5925	* @returns VBox status code.
5926	* @param pVM The cross context VM structure.
5927	* @param idChunk The chunk in question.
5928	* @param ppChunk Where to store the chunk tracking structure.
5929	*
5930	* @remarks Called from within the PGM critical section.
5931	* @remarks Can be called from any thread!
5932	*/
5933	int pgmR3PhysChunkMap(PVM pVM, uint32_t idChunk, PPPGMCHUNKR3MAP ppChunk)
5934	{
5935	int rc;
5936
5937	PGM_LOCK_ASSERT_OWNER(pVM);
5938
5939	/*
5940	* Move the chunk time forward.
5941	*/
5942	pVM->pgm.s.ChunkR3Map.iNow++;
5943	if (pVM->pgm.s.ChunkR3Map.iNow == 0)
5944	{
5945	pVM->pgm.s.ChunkR3Map.iNow = 4;
5946	RTAvlU32DoWithAll(&pVM->pgm.s.ChunkR3Map.pTree, true /fFromLeft/, pgmR3PhysChunkAgeingRolloverCallback, NULL);
5947	}
5948
5949	/*
5950	* Allocate a new tracking structure first.
5951	*/
5952	PPGMCHUNKR3MAP pChunk = (PPGMCHUNKR3MAP)MMR3HeapAllocZ(pVM, MM_TAG_PGM_CHUNK_MAPPING, sizeof(*pChunk));
5953	AssertReturn(pChunk, VERR_NO_MEMORY);
5954	pChunk->Core.Key = idChunk;
5955	pChunk->iLastUsed = pVM->pgm.s.ChunkR3Map.iNow;
5956
5957	/*
5958	* Request the ring-0 part to map the chunk in question.
5959	*/
5960	GMMMAPUNMAPCHUNKREQ Req;
5961	Req.Hdr.u32Magic = SUPVMMR0REQHDR_MAGIC;
5962	Req.Hdr.cbReq = sizeof(Req);
5963	Req.pvR3 = NULL;
5964	Req.idChunkMap = idChunk;
5965	Req.idChunkUnmap = NIL_GMM_CHUNKID;
5966
5967	/* Must be callable from any thread, so can't use VMMR3CallR0. */
5968	STAM_PROFILE_START(&pVM->pgm.s.Stats.StatChunkMap, a);
5969	rc = SUPR3CallVMMR0Ex(VMCC_GET_VMR0_FOR_CALL(pVM), NIL_VMCPUID, VMMR0_DO_GMM_MAP_UNMAP_CHUNK, 0, &Req.Hdr);
5970	STAM_PROFILE_STOP(&pVM->pgm.s.Stats.StatChunkMap, a);
5971	if (RT_SUCCESS(rc))
5972	{
5973	pChunk->pv = Req.pvR3;
5974
5975	/*
5976	* If we're running out of virtual address space, then we should
5977	* unmap another chunk.
5978	*
5979	* Currently, an unmap operation requires that all other virtual CPUs
5980	* are idling and not by chance making use of the memory we're
5981	* unmapping. So, we create an async unmap operation here.
5982	*
5983	* Now, when creating or restoring a saved state this wont work very
5984	* well since we may want to restore all guest RAM + a little something.
5985	* So, we have to do the unmap synchronously. Fortunately for us
5986	* though, during these operations the other virtual CPUs are inactive
5987	* and it should be safe to do this.
5988	*/
5989	/** @todo Eventually we should lock all memory when used and do
5990	* map+unmap as one kernel call without any rendezvous or
5991	* other precautions. */
5992	if (pVM->pgm.s.ChunkR3Map.c + 1 >= pVM->pgm.s.ChunkR3Map.cMax)
5993	{
5994	switch (VMR3GetState(pVM))
5995	{
5996	case VMSTATE_LOADING:
5997	case VMSTATE_SAVING:
5998	{
5999	PVMCPU pVCpu = VMMGetCpu(pVM);
6000	if ( pVCpu
6001	&& pVM->pgm.s.cDeprecatedPageLocks == 0)
6002	{
6003	pgmR3PhysUnmapChunkRendezvous(pVM, pVCpu, NULL);
6004	break;
6005	}
6006	}
6007	RT_FALL_THRU();
6008	default:
6009	rc = VMR3ReqCallNoWait(pVM, VMCPUID_ANY_QUEUE, (PFNRT)pgmR3PhysUnmapChunk, 1, pVM);
6010	AssertRC(rc);
6011	break;
6012	}
6013	}
6014
6015	/*
6016	* Update the tree. We must do this after any unmapping to make sure
6017	* the chunk we're going to return isn't unmapped by accident.
6018	*/
6019	AssertPtr(Req.pvR3);
6020	bool fRc = RTAvlU32Insert(&pVM->pgm.s.ChunkR3Map.pTree, &pChunk->Core);
6021	AssertRelease(fRc);
6022	pVM->pgm.s.ChunkR3Map.c++;
6023	pVM->pgm.s.cMappedChunks++;
6024	}
6025	else
6026	{
6027	/** @todo this may fail because of /proc/sys/vm/max_map_count, so we
6028	* should probably restrict ourselves on linux. */
6029	AssertRC(rc);
6030	MMR3HeapFree(pChunk);
6031	pChunk = NULL;
6032	}
6033
6034	*ppChunk = pChunk;
6035	return rc;
6036	}
6037
6038
6039	/**
6040	* Invalidates the TLB for the ring-3 mapping cache.
6041	*
6042	* @param pVM The cross context VM structure.
6043	*/
6044	VMMR3DECL(void) PGMR3PhysChunkInvalidateTLB(PVM pVM)
6045	{
6046	PGM_LOCK_VOID(pVM);
6047	for (unsigned i = 0; i < RT_ELEMENTS(pVM->pgm.s.ChunkR3Map.Tlb.aEntries); i++)
6048	{
6049	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].idChunk = NIL_GMM_CHUNKID;
6050	pVM->pgm.s.ChunkR3Map.Tlb.aEntries[i].pChunk = NULL;
6051	}
6052	/* The page map TLB references chunks, so invalidate that one too. */
6053	pgmPhysInvalidatePageMapTLB(pVM);
6054	PGM_UNLOCK(pVM);
6055	}
6056
6057
6058	/**
6059	* Response to VM_FF_PGM_NEED_HANDY_PAGES and helper for pgmPhysEnsureHandyPage.
6060	*
6061	* This function will also work the VM_FF_PGM_NO_MEMORY force action flag, to
6062	* signal and clear the out of memory condition. When called, this API is used
6063	* to try clear the condition when the user wants to resume.
6064	*
6065	* @returns The following VBox status codes.
6066	* @retval VINF_SUCCESS on success. FFs cleared.
6067	* @retval VINF_EM_NO_MEMORY if we're out of memory. The FF is not cleared in
6068	* this case and it gets accompanied by VM_FF_PGM_NO_MEMORY.
6069	*
6070	* @param pVM The cross context VM structure.
6071	*
6072	* @remarks The VINF_EM_NO_MEMORY status is for the benefit of the FF processing
6073	* in EM.cpp and shouldn't be propagated outside TRPM, HM, EM and
6074	* pgmPhysEnsureHandyPage. There is one exception to this in the \#PF
6075	* handler.
6076	*/
6077	VMMR3DECL(int) PGMR3PhysAllocateHandyPages(PVM pVM)
6078	{
6079	PGM_LOCK_VOID(pVM);
6080
6081	/*
6082	* Allocate more pages, noting down the index of the first new page.
6083	*/
6084	uint32_t iClear = pVM->pgm.s.cHandyPages;
6085	AssertMsgReturn(iClear <= RT_ELEMENTS(pVM->pgm.s.aHandyPages), ("%d", iClear), VERR_PGM_HANDY_PAGE_IPE);
6086	Log(("PGMR3PhysAllocateHandyPages: %d -> %d\n", iClear, RT_ELEMENTS(pVM->pgm.s.aHandyPages)));
6087	int rc = VMMR3CallR0(pVM, VMMR0_DO_PGM_ALLOCATE_HANDY_PAGES, 0, NULL);
6088	/** @todo we should split this up into an allocate and flush operation. sometimes you want to flush and not allocate more (which will trigger the vm account limit error) */
6089	if ( rc == VERR_GMM_HIT_VM_ACCOUNT_LIMIT
6090	&& pVM->pgm.s.cHandyPages > 0)
6091	{
6092	/* Still handy pages left, so don't panic. */
6093	rc = VINF_SUCCESS;
6094	}
6095
6096	if (RT_SUCCESS(rc))
6097	{
6098	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
6099	Assert(pVM->pgm.s.cHandyPages > 0);
6100	#ifdef VBOX_STRICT
6101	uint32_t i;
6102	for (i = iClear; i < pVM->pgm.s.cHandyPages; i++)
6103	if ( pVM->pgm.s.aHandyPages[i].idPage == NIL_GMM_PAGEID
6104	\|\| pVM->pgm.s.aHandyPages[i].idSharedPage != NIL_GMM_PAGEID
6105	\|\| (pVM->pgm.s.aHandyPages[i].HCPhysGCPhys & GUEST_PAGE_OFFSET_MASK))
6106	break;
6107	if (i != pVM->pgm.s.cHandyPages)
6108	{
6109	RTAssertMsg1Weak(NULL, __LINE__, __FILE__, __FUNCTION__);
6110	RTAssertMsg2Weak("i=%d iClear=%d cHandyPages=%d\n", i, iClear, pVM->pgm.s.cHandyPages);
6111	for (uint32_t j = iClear; j < pVM->pgm.s.cHandyPages; j++)
6112	RTAssertMsg2Add("%03d: idPage=%d HCPhysGCPhys=%RHp idSharedPage=%d%s\n", j,
6113	pVM->pgm.s.aHandyPages[j].idPage,
6114	pVM->pgm.s.aHandyPages[j].HCPhysGCPhys,
6115	pVM->pgm.s.aHandyPages[j].idSharedPage,
6116	j == i ? " <---" : "");
6117	RTAssertPanic();
6118	}
6119	#endif
6120	}
6121	else
6122	{
6123	/*
6124	* We should never get here unless there is a genuine shortage of
6125	* memory (or some internal error). Flag the error so the VM can be
6126	* suspended ASAP and the user informed. If we're totally out of
6127	* handy pages we will return failure.
6128	*/
6129	/* Report the failure. */
6130	LogRel(("PGM: Failed to procure handy pages; rc=%Rrc cHandyPages=%#x\n"
6131	" cAllPages=%#x cPrivatePages=%#x cSharedPages=%#x cZeroPages=%#x\n",
6132	rc, pVM->pgm.s.cHandyPages,
6133	pVM->pgm.s.cAllPages, pVM->pgm.s.cPrivatePages, pVM->pgm.s.cSharedPages, pVM->pgm.s.cZeroPages));
6134
6135	if ( rc != VERR_NO_MEMORY
6136	&& rc != VERR_NO_PHYS_MEMORY
6137	&& rc != VERR_LOCK_FAILED)
6138	for (uint32_t i = 0; i < RT_ELEMENTS(pVM->pgm.s.aHandyPages); i++)
6139	{
6140	LogRel(("PGM: aHandyPages[#%#04x] = {.HCPhysGCPhys=%RHp, .idPage=%#08x, .idSharedPage=%#08x}\n",
6141	i, pVM->pgm.s.aHandyPages[i].HCPhysGCPhys, pVM->pgm.s.aHandyPages[i].idPage,
6142	pVM->pgm.s.aHandyPages[i].idSharedPage));
6143	uint32_t const idPage = pVM->pgm.s.aHandyPages[i].idPage;
6144	if (idPage != NIL_GMM_PAGEID)
6145	{
6146	uint32_t const idRamRangeMax = RT_MIN(pVM->pgm.s.idRamRangeMax, RT_ELEMENTS(pVM->pgm.s.apRamRanges) - 1U);
6147	for (uint32_t idRamRange = 0; idRamRange <= idRamRangeMax; idRamRange++)
6148	{
6149	PPGMRAMRANGE const pRam = pVM->pgm.s.apRamRanges[idRamRange];
6150	Assert(pRam \|\| idRamRange == 0);
6151	if (!pRam) continue;
6152	Assert(pRam->idRange == idRamRange);
6153
6154	uint32_t const cPages = pRam->cb >> GUEST_PAGE_SHIFT;
6155	for (uint32_t iPage = 0; iPage < cPages; iPage++)
6156	if (PGM_PAGE_GET_PAGEID(&pRam->aPages[iPage]) == idPage)
6157	LogRel(("PGM: Used by %RGp %R[pgmpage] (%s)\n",
6158	pRam->GCPhys + ((RTGCPHYS)iPage << GUEST_PAGE_SHIFT), &pRam->aPages[iPage], pRam->pszDesc));
6159	}
6160	}
6161	}
6162
6163	if (rc == VERR_NO_MEMORY)
6164	{
6165	uint64_t cbHostRamAvail = 0;
6166	int rc2 = RTSystemQueryAvailableRam(&cbHostRamAvail);
6167	if (RT_SUCCESS(rc2))
6168	LogRel(("Host RAM: %RU64MB available\n", cbHostRamAvail / _1M));
6169	else
6170	LogRel(("Cannot determine the amount of available host memory\n"));
6171	}
6172
6173	/* Set the FFs and adjust rc. */
6174	VM_FF_SET(pVM, VM_FF_PGM_NEED_HANDY_PAGES);
6175	VM_FF_SET(pVM, VM_FF_PGM_NO_MEMORY);
6176	if ( rc == VERR_NO_MEMORY
6177	\|\| rc == VERR_NO_PHYS_MEMORY
6178	\|\| rc == VERR_LOCK_FAILED)
6179	rc = VINF_EM_NO_MEMORY;
6180	}
6181
6182	PGM_UNLOCK(pVM);
6183	return rc;
6184	}
6185
6186
6187	/*********************************************************************************************************************************
6188	* Other Stuff *
6189	*********************************************************************************************************************************/
6190
6191	#if !defined(VBOX_VMM_TARGET_ARMV8)
6192	/**
6193	* Sets the Address Gate 20 state.
6194	*
6195	* @param pVCpu The cross context virtual CPU structure.
6196	* @param fEnable True if the gate should be enabled.
6197	* False if the gate should be disabled.
6198	*/
6199	VMMDECL(void) PGMR3PhysSetA20(PVMCPU pVCpu, bool fEnable)
6200	{
6201	LogFlow(("PGMR3PhysSetA20 %d (was %d)\n", fEnable, pVCpu->pgm.s.fA20Enabled));
6202	if (pVCpu->pgm.s.fA20Enabled != fEnable)
6203	{
6204	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6205	PCCPUMCTX pCtx = CPUMQueryGuestCtxPtr(pVCpu);
6206	if ( CPUMIsGuestInVmxRootMode(pCtx)
6207	&& !fEnable)
6208	{
6209	Log(("Cannot enter A20M mode while in VMX root mode\n"));
6210	return;
6211	}
6212	#endif
6213	pVCpu->pgm.s.fA20Enabled = fEnable;
6214	pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!fEnable << 20);
6215	if (VM_IS_NEM_ENABLED(pVCpu->CTX_SUFF(pVM)))
6216	NEMR3NotifySetA20(pVCpu, fEnable);
6217	#ifdef PGM_WITH_A20
6218	VMCPU_FF_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
6219	pgmR3RefreshShadowModeAfterA20Change(pVCpu);
6220	HMFlushTlb(pVCpu);
6221	#endif
6222	#if 0 /* PGMGetPage will apply the A20 mask to the GCPhys it returns, so we must invalid both sides of the TLB. */
6223	IEMTlbInvalidateAllPhysical(pVCpu);
6224	#else
6225	IEMTlbInvalidateAll(pVCpu);
6226	#endif
6227	STAM_REL_COUNTER_INC(&pVCpu->pgm.s.cA20Changes);
6228	}
6229	}
6230	#endif
6231

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/PGMPhys.cpp@ 104840

Download in other formats: