exec.c@ 36211

Last change on this file since 36211 was 36211, checked in by vboxsync, 14 years ago
replaced kvm.h with our own which stubbs the exposed functions.
Property svn:eol-style set to `native`
File size: 124.7 KB

Line
1	/*
2	* virtual page mapping and translated block handling
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	/*
21	* Oracle LGPL Disclaimer: For the avoidance of doubt, except that if any license choice
22	* other than GPL or LGPL is available it will apply instead, Oracle elects to use only
23	* the Lesser General Public License version 2.1 (LGPLv2) at this time for any software where
24	* a choice of LGPL license versions is made available with the language indicating
25	* that LGPLv2 or any later version may be used, or where a choice of which version
26	* of the LGPL is applied is otherwise unspecified.
27	*/
28
29	#include "config.h"
30	#ifndef VBOX
31	#ifdef _WIN32
32	#include <windows.h>
33	#else
34	#include <sys/types.h>
35	#include <sys/mman.h>
36	#endif
37	#include <stdlib.h>
38	#include <stdio.h>
39	#include <stdarg.h>
40	#include <string.h>
41	#include <errno.h>
42	#include <unistd.h>
43	#include <inttypes.h>
44	#else /* VBOX */
45	# include <stdlib.h>
46	# include <stdio.h>
47	# include <iprt/alloc.h>
48	# include <iprt/string.h>
49	# include <iprt/param.h>
50	# include <VBox/vmm/pgm.h> /* PGM_DYNAMIC_RAM_ALLOC */
51	#endif /* VBOX */
52
53	#include "cpu.h"
54	#include "exec-all.h"
55	#include "qemu-common.h"
56	#include "tcg.h"
57	#ifndef VBOX
58	#include "hw/hw.h"
59	#endif
60	#include "osdep.h"
61	#include "kvm.h"
62	#if defined(CONFIG_USER_ONLY)
63	#include <qemu.h>
64	#endif
65
66	//#define DEBUG_TB_INVALIDATE
67	//#define DEBUG_FLUSH
68	//#define DEBUG_TLB
69	//#define DEBUG_UNASSIGNED
70
71	/* make various TB consistency checks */
72	//#define DEBUG_TB_CHECK
73	//#define DEBUG_TLB_CHECK
74
75	//#define DEBUG_IOPORT
76	//#define DEBUG_SUBPAGE
77
78	#if !defined(CONFIG_USER_ONLY)
79	/* TB consistency checks only implemented for usermode emulation. */
80	#undef DEBUG_TB_CHECK
81	#endif
82
83	#define SMC_BITMAP_USE_THRESHOLD 10
84
85	#if defined(TARGET_SPARC64)
86	#define TARGET_PHYS_ADDR_SPACE_BITS 41
87	#elif defined(TARGET_SPARC)
88	#define TARGET_PHYS_ADDR_SPACE_BITS 36
89	#elif defined(TARGET_ALPHA)
90	#define TARGET_PHYS_ADDR_SPACE_BITS 42
91	#define TARGET_VIRT_ADDR_SPACE_BITS 42
92	#elif defined(TARGET_PPC64)
93	#define TARGET_PHYS_ADDR_SPACE_BITS 42
94	#elif defined(TARGET_X86_64) && !defined(CONFIG_KQEMU)
95	#define TARGET_PHYS_ADDR_SPACE_BITS 42
96	#elif defined(TARGET_I386) && !defined(CONFIG_KQEMU)
97	#define TARGET_PHYS_ADDR_SPACE_BITS 36
98	#else
99	/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
100	#define TARGET_PHYS_ADDR_SPACE_BITS 32
101	#endif
102
103	static TranslationBlock *tbs;
104	int code_gen_max_blocks;
105	TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
106	static int nb_tbs;
107	/* any access to the tbs or the page table must use this lock */
108	spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
109
110	#ifndef VBOX
111	#if defined(__arm__) \|\| defined(__sparc_v9__)
112	/* The prologue must be reachable with a direct jump. ARM and Sparc64
113	have limited branch ranges (possibly also PPC) so place it in a
114	section close to code segment. */
115	#define code_gen_section \
116	__attribute__((__section__(".gen_code"))) \
117	__attribute__((aligned (32)))
118	#elif defined(_WIN32)
119	/* Maximum alignment for Win32 is 16. */
120	#define code_gen_section \
121	__attribute__((aligned (16)))
122	#else
123	#define code_gen_section \
124	__attribute__((aligned (32)))
125	#endif
126
127	uint8_t code_gen_prologue[1024] code_gen_section;
128	#else /* VBOX */
129	extern uint8_t* code_gen_prologue;
130	#endif /* VBOX */
131	static uint8_t *code_gen_buffer;
132	static unsigned long code_gen_buffer_size;
133	/* threshold to flush the translated code buffer */
134	static unsigned long code_gen_buffer_max_size;
135	uint8_t *code_gen_ptr;
136
137	#ifndef VBOX
138	#if !defined(CONFIG_USER_ONLY)
139	int phys_ram_fd;
140	uint8_t *phys_ram_dirty;
141	static int in_migration;
142
143	typedef struct RAMBlock {
144	uint8_t *host;
145	ram_addr_t offset;
146	ram_addr_t length;
147	struct RAMBlock *next;
148	} RAMBlock;
149
150	static RAMBlock *ram_blocks;
151	/* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
152	then we can no longer assume contiguous ram offsets, and external uses
153	of this variable will break. */
154	ram_addr_t last_ram_offset;
155	#endif
156	#else /* VBOX */
157	/* we have memory ranges (the high PC-BIOS mapping) which
158	causes some pages to fall outside the dirty map here. */
159	RTGCPHYS phys_ram_dirty_size;
160	uint8_t *phys_ram_dirty;
161	#endif /* VBOX */
162
163	CPUState *first_cpu;
164	/* current CPU in the current thread. It is only valid inside
165	cpu_exec() */
166	CPUState *cpu_single_env;
167	/* 0 = Do not count executed instructions.
168	1 = Precise instruction counting.
169	2 = Adaptive rate instruction counting. */
170	int use_icount = 0;
171	/* Current instruction counter. While executing translated code this may
172	include some instructions that have not yet been executed. */
173	int64_t qemu_icount;
174
175	typedef struct PageDesc {
176	/* list of TBs intersecting this ram page */
177	TranslationBlock *first_tb;
178	/* in order to optimize self modifying code, we count the number
179	of lookups we do to a given page to use a bitmap */
180	unsigned int code_write_count;
181	uint8_t *code_bitmap;
182	#if defined(CONFIG_USER_ONLY)
183	unsigned long flags;
184	#endif
185	} PageDesc;
186
187	typedef struct PhysPageDesc {
188	/* offset in host memory of the page + io_index in the low bits */
189	ram_addr_t phys_offset;
190	ram_addr_t region_offset;
191	} PhysPageDesc;
192
193	#define L2_BITS 10
194	#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
195	/* XXX: this is a temporary hack for alpha target.
196	* In the future, this is to be replaced by a multi-level table
197	* to actually be able to handle the complete 64 bits address space.
198	*/
199	#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
200	#else
201	#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
202	#endif
203	#ifdef VBOX
204	#define L0_BITS (TARGET_PHYS_ADDR_SPACE_BITS - 32)
205	#endif
206
207	#ifdef VBOX
208	#define L0_SIZE (1 << L0_BITS)
209	#endif
210	#define L1_SIZE (1 << L1_BITS)
211	#define L2_SIZE (1 << L2_BITS)
212
213	unsigned long qemu_real_host_page_size;
214	unsigned long qemu_host_page_bits;
215	unsigned long qemu_host_page_size;
216	unsigned long qemu_host_page_mask;
217
218	/* XXX: for system emulation, it could just be an array */
219	#ifndef VBOX
220	static PageDesc *l1_map[L1_SIZE];
221	static PhysPageDesc **l1_phys_map;
222	#else
223	static unsigned l0_map_max_used = 0;
224	static PageDesc **l0_map[L0_SIZE];
225	static void **l0_phys_map[L0_SIZE];
226	#endif
227
228	#if !defined(CONFIG_USER_ONLY)
229	static void io_mem_init(void);
230
231	/* io memory support */
232	CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
233	CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
234	void *io_mem_opaque[IO_MEM_NB_ENTRIES];
235	static char io_mem_used[IO_MEM_NB_ENTRIES];
236	static int io_mem_watch;
237	#endif
238
239	#ifndef VBOX
240	/* log support */
241	static const char *logfilename = "/tmp/qemu.log";
242	#endif /* !VBOX */
243	FILE *logfile;
244	int loglevel;
245	#ifndef VBOX
246	static int log_append = 0;
247	#endif
248
249	/* statistics */
250	#ifndef VBOX
251	static int tlb_flush_count;
252	static int tb_flush_count;
253	static int tb_phys_invalidate_count;
254	#else /* VBOX - Resettable U32 stats, see VBoxRecompiler.c. */
255	uint32_t tlb_flush_count;
256	uint32_t tb_flush_count;
257	uint32_t tb_phys_invalidate_count;
258	#endif /* VBOX */
259
260	#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
261	typedef struct subpage_t {
262	target_phys_addr_t base;
263	CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
264	CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
265	void *opaque[TARGET_PAGE_SIZE][2][4];
266	ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
267	} subpage_t;
268
269	#ifndef VBOX
270	#ifdef _WIN32
271	static void map_exec(void *addr, long size)
272	{
273	DWORD old_protect;
274	VirtualProtect(addr, size,
275	PAGE_EXECUTE_READWRITE, &old_protect);
276
277	}
278	#else
279	static void map_exec(void *addr, long size)
280	{
281	unsigned long start, end, page_size;
282
283	page_size = getpagesize();
284	start = (unsigned long)addr;
285	start &= ~(page_size - 1);
286
287	end = (unsigned long)addr + size;
288	end += page_size - 1;
289	end &= ~(page_size - 1);
290
291	mprotect((void *)start, end - start,
292	PROT_READ \| PROT_WRITE \| PROT_EXEC);
293	}
294	#endif
295	#else /* VBOX */
296	static void map_exec(void *addr, long size)
297	{
298	RTMemProtect(addr, size,
299	RTMEM_PROT_EXEC \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
300	}
301	#endif /* VBOX */
302
303	static void page_init(void)
304	{
305	/* NOTE: we can always suppose that qemu_host_page_size >=
306	TARGET_PAGE_SIZE */
307	#ifdef VBOX
308	RTMemProtect(code_gen_buffer, sizeof(code_gen_buffer),
309	RTMEM_PROT_EXEC \| RTMEM_PROT_READ \| RTMEM_PROT_WRITE);
310	qemu_real_host_page_size = PAGE_SIZE;
311	#else /* !VBOX */
312	#ifdef _WIN32
313	{
314	SYSTEM_INFO system_info;
315
316	GetSystemInfo(&system_info);
317	qemu_real_host_page_size = system_info.dwPageSize;
318	}
319	#else
320	qemu_real_host_page_size = getpagesize();
321	#endif
322	#endif /* !VBOX */
323	if (qemu_host_page_size == 0)
324	qemu_host_page_size = qemu_real_host_page_size;
325	if (qemu_host_page_size < TARGET_PAGE_SIZE)
326	qemu_host_page_size = TARGET_PAGE_SIZE;
327	qemu_host_page_bits = 0;
328	#ifndef VBOX
329	while ((1 << qemu_host_page_bits) < qemu_host_page_size)
330	#else
331	while ((1 << qemu_host_page_bits) < (int)qemu_host_page_size)
332	#endif
333	qemu_host_page_bits++;
334	qemu_host_page_mask = ~(qemu_host_page_size - 1);
335	#ifndef VBOX
336	l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
337	memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
338	#endif
339
340	#ifdef VBOX
341	/* We use other means to set reserved bit on our pages */
342	#else /* !VBOX */
343	#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
344	{
345	long long startaddr, endaddr;
346	FILE *f;
347	int n;
348
349	mmap_lock();
350	last_brk = (unsigned long)sbrk(0);
351	f = fopen("/proc/self/maps", "r");
352	if (f) {
353	do {
354	n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
355	if (n == 2) {
356	startaddr = MIN(startaddr,
357	(1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
358	endaddr = MIN(endaddr,
359	(1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
360	page_set_flags(startaddr & TARGET_PAGE_MASK,
361	TARGET_PAGE_ALIGN(endaddr),
362	PAGE_RESERVED);
363	}
364	} while (!feof(f));
365	fclose(f);
366	}
367	mmap_unlock();
368	}
369	#endif
370	#endif /* !VBOX */
371	}
372
373	static inline PageDesc **page_l1_map(target_ulong index)
374	{
375	#ifndef VBOX
376	#if TARGET_LONG_BITS > 32
377	/* Host memory outside guest VM. For 32-bit targets we have already
378	excluded high addresses. */
379	if (index > ((target_ulong)L2_SIZE * L1_SIZE))
380	return NULL;
381	#endif
382	return &l1_map[index >> L2_BITS];
383	#else /* VBOX */
384	PageDesc **l1_map;
385	AssertMsgReturn(index < (target_ulong)L2_SIZE * L1_SIZE * L0_SIZE,
386	("index=%RGp >= %RGp; L1_SIZE=%#x L2_SIZE=%#x L0_SIZE=%#x\n",
387	(RTGCPHYS)index, (RTGCPHYS)L2_SIZE * L1_SIZE, L1_SIZE, L2_SIZE, L0_SIZE),
388	NULL);
389	l1_map = l0_map[index >> (L1_BITS + L2_BITS)];
390	if (RT_UNLIKELY(!l1_map))
391	{
392	unsigned i0 = index >> (L1_BITS + L2_BITS);
393	l0_map[i0] = l1_map = qemu_mallocz(sizeof(PageDesc ) L1_SIZE);
394	if (RT_UNLIKELY(!l1_map))
395	return NULL;
396	if (i0 >= l0_map_max_used)
397	l0_map_max_used = i0 + 1;
398	}
399	return &l1_map[(index >> L2_BITS) & (L1_SIZE - 1)];
400	#endif /* VBOX */
401	}
402
403	static inline PageDesc *page_find_alloc(target_ulong index)
404	{
405	PageDesc *lp, p;
406	lp = page_l1_map(index);
407	if (!lp)
408	return NULL;
409
410	p = *lp;
411	if (!p) {
412	/* allocate if not found */
413	#if defined(CONFIG_USER_ONLY)
414	size_t len = sizeof(PageDesc) * L2_SIZE;
415	/* Don't use qemu_malloc because it may recurse. */
416	p = mmap(NULL, len, PROT_READ \| PROT_WRITE,
417	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
418	*lp = p;
419	if (h2g_valid(p)) {
420	unsigned long addr = h2g(p);
421	page_set_flags(addr & TARGET_PAGE_MASK,
422	TARGET_PAGE_ALIGN(addr + len),
423	PAGE_RESERVED);
424	}
425	#else
426	p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
427	*lp = p;
428	#endif
429	}
430	return p + (index & (L2_SIZE - 1));
431	}
432
433	static inline PageDesc *page_find(target_ulong index)
434	{
435	PageDesc *lp, p;
436	lp = page_l1_map(index);
437	if (!lp)
438	return NULL;
439
440	p = *lp;
441	if (!p) {
442	return NULL;
443	}
444	return p + (index & (L2_SIZE - 1));
445	}
446
447	static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
448	{
449	void lp, p;
450	PhysPageDesc *pd;
451
452	#ifndef VBOX
453	p = (void **)l1_phys_map;
454	#if TARGET_PHYS_ADDR_SPACE_BITS > 32
455
456	#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
457	#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
458	#endif
459	lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
460	p = *lp;
461	if (!p) {
462	/* allocate if not found */
463	if (!alloc)
464	return NULL;
465	p = qemu_vmalloc(sizeof(void ) L1_SIZE);
466	memset(p, 0, sizeof(void ) L1_SIZE);
467	*lp = p;
468	}
469	#endif
470	#else /* VBOX */
471	/* level 0 lookup and lazy allocation of level 1 map. */
472	if (RT_UNLIKELY(index >= (target_phys_addr_t)L2_SIZE * L1_SIZE * L0_SIZE))
473	return NULL;
474	p = l0_phys_map[index >> (L1_BITS + L2_BITS)];
475	if (RT_UNLIKELY(!p)) {
476	if (!alloc)
477	return NULL;
478	p = qemu_vmalloc(sizeof(void *) L1_SIZE);
479	memset(p, 0, sizeof(void *) L1_SIZE);
480	l0_phys_map[index >> (L1_BITS + L2_BITS)] = p;
481	}
482
483	/* level 1 lookup and lazy allocation of level 2 map. */
484	#endif /* VBOX */
485	lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
486	pd = *lp;
487	if (!pd) {
488	int i;
489	/* allocate if not found */
490	if (!alloc)
491	return NULL;
492	pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
493	*lp = pd;
494	for (i = 0; i < L2_SIZE; i++) {
495	pd[i].phys_offset = IO_MEM_UNASSIGNED;
496	pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
497	}
498	}
499	return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
500	}
501
502	static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
503	{
504	return phys_page_find_alloc(index, 0);
505	}
506
507	#if !defined(CONFIG_USER_ONLY)
508	static void tlb_protect_code(ram_addr_t ram_addr);
509	static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
510	target_ulong vaddr);
511	#define mmap_lock() do { } while(0)
512	#define mmap_unlock() do { } while(0)
513	#endif
514
515	#ifdef VBOX /* We don't need such huge codegen buffer size, as execute
516	most of the code in raw or hwacc mode. */
517	#define DEFAULT_CODE_GEN_BUFFER_SIZE (8 * 1024 * 1024)
518	#else /* !VBOX */
519	#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
520	#endif /* !VBOX */
521
522	#if defined(CONFIG_USER_ONLY)
523	/* Currently it is not recommended to allocate big chunks of data in
524	user mode. It will change when a dedicated libc will be used */
525	#define USE_STATIC_CODE_GEN_BUFFER
526	#endif
527
528	#if defined(VBOX) && defined(USE_STATIC_CODE_GEN_BUFFER)
529	# error "VBox allocates codegen buffer dynamically"
530	#endif
531
532	#ifdef USE_STATIC_CODE_GEN_BUFFER
533	static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
534	#endif
535
536	static void code_gen_alloc(unsigned long tb_size)
537	{
538	#ifdef USE_STATIC_CODE_GEN_BUFFER
539	code_gen_buffer = static_code_gen_buffer;
540	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
541	map_exec(code_gen_buffer, code_gen_buffer_size);
542	#else
543	# ifdef VBOX
544	/* We cannot use phys_ram_size here, as it's 0 now,
545	* it only gets initialized once RAM registration callback
546	* (REMR3NotifyPhysRamRegister()) called.
547	*/
548	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
549	# else /* !VBOX */
550	code_gen_buffer_size = tb_size;
551	if (code_gen_buffer_size == 0) {
552	#if defined(CONFIG_USER_ONLY)
553	/* in user mode, phys_ram_size is not meaningful */
554	code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
555	#else
556	/* XXX: needs ajustments */
557	code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
558	#endif
559	}
560	if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
561	code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
562	# endif /* !VBOX */
563	/* The code gen buffer location may have constraints depending on
564	the host cpu and OS */
565	# ifdef VBOX
566	code_gen_buffer = RTMemExecAlloc(code_gen_buffer_size);
567
568	if (!code_gen_buffer) {
569	LogRel(("REM: failed allocate codegen buffer %lld\n",
570	code_gen_buffer_size));
571	return;
572	}
573	# else /* !VBOX */
574	#if defined(__linux__)
575	{
576	int flags;
577	void *start = NULL;
578
579	flags = MAP_PRIVATE \| MAP_ANONYMOUS;
580	#if defined(__x86_64__)
581	flags \|= MAP_32BIT;
582	/* Cannot map more than that */
583	if (code_gen_buffer_size > (800 * 1024 * 1024))
584	code_gen_buffer_size = (800 * 1024 * 1024);
585	#elif defined(__sparc_v9__)
586	// Map the buffer below 2G, so we can use direct calls and branches
587	flags \|= MAP_FIXED;
588	start = (void *) 0x60000000UL;
589	if (code_gen_buffer_size > (512 * 1024 * 1024))
590	code_gen_buffer_size = (512 * 1024 * 1024);
591	#elif defined(__arm__)
592	/* Map the buffer below 32M, so we can use direct calls and branches */
593	flags \|= MAP_FIXED;
594	start = (void *) 0x01000000UL;
595	if (code_gen_buffer_size > 16 * 1024 * 1024)
596	code_gen_buffer_size = 16 * 1024 * 1024;
597	#endif
598	code_gen_buffer = mmap(start, code_gen_buffer_size,
599	PROT_WRITE \| PROT_READ \| PROT_EXEC,
600	flags, -1, 0);
601	if (code_gen_buffer == MAP_FAILED) {
602	fprintf(stderr, "Could not allocate dynamic translator buffer\n");
603	exit(1);
604	}
605	}
606	#elif defined(__FreeBSD__) \|\| defined(__DragonFly__)
607	{
608	int flags;
609	void *addr = NULL;
610	flags = MAP_PRIVATE \| MAP_ANONYMOUS;
611	#if defined(__x86_64__)
612	/* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
613	* 0x40000000 is free */
614	flags \|= MAP_FIXED;
615	addr = (void *)0x40000000;
616	/* Cannot map more than that */
617	if (code_gen_buffer_size > (800 * 1024 * 1024))
618	code_gen_buffer_size = (800 * 1024 * 1024);
619	#endif
620	code_gen_buffer = mmap(addr, code_gen_buffer_size,
621	PROT_WRITE \| PROT_READ \| PROT_EXEC,
622	flags, -1, 0);
623	if (code_gen_buffer == MAP_FAILED) {
624	fprintf(stderr, "Could not allocate dynamic translator buffer\n");
625	exit(1);
626	}
627	}
628	#else
629	code_gen_buffer = qemu_malloc(code_gen_buffer_size);
630	map_exec(code_gen_buffer, code_gen_buffer_size);
631	#endif
632	# endif /* !VBOX */
633	#endif /* !USE_STATIC_CODE_GEN_BUFFER */
634	#ifndef VBOX
635	map_exec(code_gen_prologue, sizeof(code_gen_prologue));
636	#else
637	map_exec(code_gen_prologue, _1K);
638	#endif
639	code_gen_buffer_max_size = code_gen_buffer_size -
640	code_gen_max_block_size();
641	code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
642	tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
643	}
644
645	/* Must be called before using the QEMU cpus. 'tb_size' is the size
646	(in bytes) allocated to the translation buffer. Zero means default
647	size. */
648	void cpu_exec_init_all(unsigned long tb_size)
649	{
650	cpu_gen_init();
651	code_gen_alloc(tb_size);
652	code_gen_ptr = code_gen_buffer;
653	page_init();
654	#if !defined(CONFIG_USER_ONLY)
655	io_mem_init();
656	#endif
657	}
658
659	#ifndef VBOX
660	#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
661
662	#define CPU_COMMON_SAVE_VERSION 1
663
664	static void cpu_common_save(QEMUFile f, void opaque)
665	{
666	CPUState *env = opaque;
667
668	cpu_synchronize_state(env, 0);
669
670	qemu_put_be32s(f, &env->halted);
671	qemu_put_be32s(f, &env->interrupt_request);
672	}
673
674	static int cpu_common_load(QEMUFile f, void opaque, int version_id)
675	{
676	CPUState *env = opaque;
677
678	if (version_id != CPU_COMMON_SAVE_VERSION)
679	return -EINVAL;
680
681	qemu_get_be32s(f, &env->halted);
682	qemu_get_be32s(f, &env->interrupt_request);
683	/* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
684	version_id is increased. */
685	env->interrupt_request &= ~0x01;
686	tlb_flush(env, 1);
687	cpu_synchronize_state(env, 1);
688
689	return 0;
690	}
691	#endif
692
693	CPUState *qemu_get_cpu(int cpu)
694	{
695	CPUState *env = first_cpu;
696
697	while (env) {
698	if (env->cpu_index == cpu)
699	break;
700	env = env->next_cpu;
701	}
702
703	return env;
704	}
705
706	#endif /* !VBOX */
707
708	void cpu_exec_init(CPUState *env)
709	{
710	CPUState **penv;
711	int cpu_index;
712
713	#if defined(CONFIG_USER_ONLY)
714	cpu_list_lock();
715	#endif
716	env->next_cpu = NULL;
717	penv = &first_cpu;
718	cpu_index = 0;
719	while (*penv != NULL) {
720	penv = &(*penv)->next_cpu;
721	cpu_index++;
722	}
723	env->cpu_index = cpu_index;
724	env->numa_node = 0;
725	TAILQ_INIT(&env->breakpoints);
726	TAILQ_INIT(&env->watchpoints);
727	*penv = env;
728	#ifndef VBOX
729	#if defined(CONFIG_USER_ONLY)
730	cpu_list_unlock();
731	#endif
732	#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
733	register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
734	cpu_common_save, cpu_common_load, env);
735	register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
736	cpu_save, cpu_load, env);
737	#endif
738	#endif /* !VBOX */
739	}
740
741	static inline void invalidate_page_bitmap(PageDesc *p)
742	{
743	if (p->code_bitmap) {
744	qemu_free(p->code_bitmap);
745	p->code_bitmap = NULL;
746	}
747	p->code_write_count = 0;
748	}
749
750	/* set to NULL all the 'first_tb' fields in all PageDescs */
751	static void page_flush_tb(void)
752	{
753	int i, j;
754	PageDesc *p;
755	#ifdef VBOX
756	int k;
757	#endif
758
759	#ifdef VBOX
760	k = l0_map_max_used;
761	while (k-- > 0) {
762	PageDesc **l1_map = l0_map[k];
763	if (l1_map) {
764	#endif
765	for(i = 0; i < L1_SIZE; i++) {
766	p = l1_map[i];
767	if (p) {
768	for(j = 0; j < L2_SIZE; j++) {
769	p->first_tb = NULL;
770	invalidate_page_bitmap(p);
771	p++;
772	}
773	}
774	}
775	#ifdef VBOX
776	}
777	}
778	#endif
779	}
780
781	/* flush all the translation blocks */
782	/* XXX: tb_flush is currently not thread safe */
783	void tb_flush(CPUState *env1)
784	{
785	CPUState *env;
786	#ifdef VBOX
787	STAM_PROFILE_START(&env1->StatTbFlush, a);
788	#endif
789	#if defined(DEBUG_FLUSH)
790	printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
791	(unsigned long)(code_gen_ptr - code_gen_buffer),
792	nb_tbs, nb_tbs > 0 ?
793	((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
794	#endif
795	if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
796	cpu_abort(env1, "Internal error: code buffer overflow\n");
797
798	nb_tbs = 0;
799
800	for(env = first_cpu; env != NULL; env = env->next_cpu) {
801	memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
802	}
803
804	memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
805	page_flush_tb();
806
807	code_gen_ptr = code_gen_buffer;
808	/* XXX: flush processor icache at this point if cache flush is
809	expensive */
810	tb_flush_count++;
811	#ifdef VBOX
812	STAM_PROFILE_STOP(&env1->StatTbFlush, a);
813	#endif
814	}
815
816	#ifdef DEBUG_TB_CHECK
817
818	static void tb_invalidate_check(target_ulong address)
819	{
820	TranslationBlock *tb;
821	int i;
822	address &= TARGET_PAGE_MASK;
823	for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
824	for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
825	if (!(address + TARGET_PAGE_SIZE <= tb->pc \|\|
826	address >= tb->pc + tb->size)) {
827	printf("ERROR invalidate: address=" TARGET_FMT_lx
828	" PC=%08lx size=%04x\n",
829	address, (long)tb->pc, tb->size);
830	}
831	}
832	}
833	}
834
835	/* verify that all the pages have correct rights for code */
836	static void tb_page_check(void)
837	{
838	TranslationBlock *tb;
839	int i, flags1, flags2;
840
841	for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
842	for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
843	flags1 = page_get_flags(tb->pc);
844	flags2 = page_get_flags(tb->pc + tb->size - 1);
845	if ((flags1 & PAGE_WRITE) \|\| (flags2 & PAGE_WRITE)) {
846	printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
847	(long)tb->pc, tb->size, flags1, flags2);
848	}
849	}
850	}
851	}
852
853	#endif
854
855	/* invalidate one TB */
856	static inline void tb_remove(TranslationBlock *ptb, TranslationBlock tb,
857	int next_offset)
858	{
859	TranslationBlock *tb1;
860	for(;;) {
861	tb1 = *ptb;
862	if (tb1 == tb) {
863	ptb = (TranslationBlock *)((char )tb1 + next_offset);
864	break;
865	}
866	ptb = (TranslationBlock *)((char )tb1 + next_offset);
867	}
868	}
869
870	static inline void tb_page_remove(TranslationBlock *ptb, TranslationBlock tb)
871	{
872	TranslationBlock *tb1;
873	unsigned int n1;
874
875	for(;;) {
876	tb1 = *ptb;
877	n1 = (long)tb1 & 3;
878	tb1 = (TranslationBlock *)((long)tb1 & ~3);
879	if (tb1 == tb) {
880	*ptb = tb1->page_next[n1];
881	break;
882	}
883	ptb = &tb1->page_next[n1];
884	}
885	}
886
887	static inline void tb_jmp_remove(TranslationBlock *tb, int n)
888	{
889	TranslationBlock tb1, *ptb;
890	unsigned int n1;
891
892	ptb = &tb->jmp_next[n];
893	tb1 = *ptb;
894	if (tb1) {
895	/* find tb(n) in circular list */
896	for(;;) {
897	tb1 = *ptb;
898	n1 = (long)tb1 & 3;
899	tb1 = (TranslationBlock *)((long)tb1 & ~3);
900	if (n1 == n && tb1 == tb)
901	break;
902	if (n1 == 2) {
903	ptb = &tb1->jmp_first;
904	} else {
905	ptb = &tb1->jmp_next[n1];
906	}
907	}
908	/* now we can suppress tb(n) from the list */
909	*ptb = tb->jmp_next[n];
910
911	tb->jmp_next[n] = NULL;
912	}
913	}
914
915	/* reset the jump entry 'n' of a TB so that it is not chained to
916	another TB */
917	static inline void tb_reset_jump(TranslationBlock *tb, int n)
918	{
919	tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
920	}
921
922	void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
923	{
924	CPUState *env;
925	PageDesc *p;
926	unsigned int h, n1;
927	target_phys_addr_t phys_pc;
928	TranslationBlock tb1, tb2;
929
930	/* remove the TB from the hash list */
931	phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
932	h = tb_phys_hash_func(phys_pc);
933	tb_remove(&tb_phys_hash[h], tb,
934	offsetof(TranslationBlock, phys_hash_next));
935
936	/* remove the TB from the page list */
937	if (tb->page_addr[0] != page_addr) {
938	p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
939	tb_page_remove(&p->first_tb, tb);
940	invalidate_page_bitmap(p);
941	}
942	if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
943	p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
944	tb_page_remove(&p->first_tb, tb);
945	invalidate_page_bitmap(p);
946	}
947
948	tb_invalidated_flag = 1;
949
950	/* remove the TB from the hash list */
951	h = tb_jmp_cache_hash_func(tb->pc);
952	for(env = first_cpu; env != NULL; env = env->next_cpu) {
953	if (env->tb_jmp_cache[h] == tb)
954	env->tb_jmp_cache[h] = NULL;
955	}
956
957	/* suppress this TB from the two jump lists */
958	tb_jmp_remove(tb, 0);
959	tb_jmp_remove(tb, 1);
960
961	/* suppress any remaining jumps to this TB */
962	tb1 = tb->jmp_first;
963	for(;;) {
964	n1 = (long)tb1 & 3;
965	if (n1 == 2)
966	break;
967	tb1 = (TranslationBlock *)((long)tb1 & ~3);
968	tb2 = tb1->jmp_next[n1];
969	tb_reset_jump(tb1, n1);
970	tb1->jmp_next[n1] = NULL;
971	tb1 = tb2;
972	}
973	tb->jmp_first = (TranslationBlock )((long)tb \| 2); / fail safe */
974
975	tb_phys_invalidate_count++;
976	}
977
978	#ifdef VBOX
979
980	void tb_invalidate_virt(CPUState *env, uint32_t eip)
981	{
982	# if 1
983	tb_flush(env);
984	# else
985	uint8_t cs_base, pc;
986	unsigned int flags, h, phys_pc;
987	TranslationBlock tb, *ptb;
988
989	flags = env->hflags;
990	flags \|= (env->eflags & (IOPL_MASK \| TF_MASK \| VM_MASK));
991	cs_base = env->segs[R_CS].base;
992	pc = cs_base + eip;
993
994	tb = tb_find(&ptb, (unsigned long)pc, (unsigned long)cs_base,
995	flags);
996
997	if(tb)
998	{
999	# ifdef DEBUG
1000	printf("invalidating TB (%08X) at %08X\n", tb, eip);
1001	# endif
1002	tb_invalidate(tb);
1003	//Note: this will leak TBs, but the whole cache will be flushed
1004	// when it happens too often
1005	tb->pc = 0;
1006	tb->cs_base = 0;
1007	tb->flags = 0;
1008	}
1009	# endif
1010	}
1011
1012	# ifdef VBOX_STRICT
1013	/**
1014	* Gets the page offset.
1015	*/
1016	unsigned long get_phys_page_offset(target_ulong addr)
1017	{
1018	PhysPageDesc *p = phys_page_find(addr >> TARGET_PAGE_BITS);
1019	return p ? p->phys_offset : 0;
1020	}
1021	# endif /* VBOX_STRICT */
1022
1023	#endif /* VBOX */
1024
1025	static inline void set_bits(uint8_t *tab, int start, int len)
1026	{
1027	int end, mask, end1;
1028
1029	end = start + len;
1030	tab += start >> 3;
1031	mask = 0xff << (start & 7);
1032	if ((start & ~7) == (end & ~7)) {
1033	if (start < end) {
1034	mask &= ~(0xff << (end & 7));
1035	*tab \|= mask;
1036	}
1037	} else {
1038	*tab++ \|= mask;
1039	start = (start + 8) & ~7;
1040	end1 = end & ~7;
1041	while (start < end1) {
1042	*tab++ = 0xff;
1043	start += 8;
1044	}
1045	if (start < end) {
1046	mask = ~(0xff << (end & 7));
1047	*tab \|= mask;
1048	}
1049	}
1050	}
1051
1052	static void build_page_bitmap(PageDesc *p)
1053	{
1054	int n, tb_start, tb_end;
1055	TranslationBlock *tb;
1056
1057	p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
1058
1059	tb = p->first_tb;
1060	while (tb != NULL) {
1061	n = (long)tb & 3;
1062	tb = (TranslationBlock *)((long)tb & ~3);
1063	/* NOTE: this is subtle as a TB may span two physical pages */
1064	if (n == 0) {
1065	/* NOTE: tb_end may be after the end of the page, but
1066	it is not a problem */
1067	tb_start = tb->pc & ~TARGET_PAGE_MASK;
1068	tb_end = tb_start + tb->size;
1069	if (tb_end > TARGET_PAGE_SIZE)
1070	tb_end = TARGET_PAGE_SIZE;
1071	} else {
1072	tb_start = 0;
1073	tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1074	}
1075	set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
1076	tb = tb->page_next[n];
1077	}
1078	}
1079
1080	TranslationBlock tb_gen_code(CPUState env,
1081	target_ulong pc, target_ulong cs_base,
1082	int flags, int cflags)
1083	{
1084	TranslationBlock *tb;
1085	uint8_t *tc_ptr;
1086	target_ulong phys_pc, phys_page2, virt_page2;
1087	int code_gen_size;
1088
1089	phys_pc = get_phys_addr_code(env, pc);
1090	tb = tb_alloc(pc);
1091	if (!tb) {
1092	/* flush must be done */
1093	tb_flush(env);
1094	/* cannot fail at this point */
1095	tb = tb_alloc(pc);
1096	/* Don't forget to invalidate previous TB info. */
1097	tb_invalidated_flag = 1;
1098	}
1099	tc_ptr = code_gen_ptr;
1100	tb->tc_ptr = tc_ptr;
1101	tb->cs_base = cs_base;
1102	tb->flags = flags;
1103	tb->cflags = cflags;
1104	cpu_gen_code(env, tb, &code_gen_size);
1105	code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1106
1107	/* check next page if needed */
1108	virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1109	phys_page2 = -1;
1110	if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1111	phys_page2 = get_phys_addr_code(env, virt_page2);
1112	}
1113	tb_link_phys(tb, phys_pc, phys_page2);
1114	return tb;
1115	}
1116
1117	/* invalidate all TBs which intersect with the target physical page
1118	starting in range [start;end[. NOTE: start and end must refer to
1119	the same physical page. 'is_cpu_write_access' should be true if called
1120	from a real cpu write access: the virtual CPU will exit the current
1121	TB if code is modified inside this TB. */
1122	void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
1123	int is_cpu_write_access)
1124	{
1125	TranslationBlock tb, tb_next, *saved_tb;
1126	CPUState *env = cpu_single_env;
1127	target_ulong tb_start, tb_end;
1128	PageDesc *p;
1129	int n;
1130	#ifdef TARGET_HAS_PRECISE_SMC
1131	int current_tb_not_found = is_cpu_write_access;
1132	TranslationBlock *current_tb = NULL;
1133	int current_tb_modified = 0;
1134	target_ulong current_pc = 0;
1135	target_ulong current_cs_base = 0;
1136	int current_flags = 0;
1137	#endif /* TARGET_HAS_PRECISE_SMC */
1138
1139	p = page_find(start >> TARGET_PAGE_BITS);
1140	if (!p)
1141	return;
1142	if (!p->code_bitmap &&
1143	++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1144	is_cpu_write_access) {
1145	/* build code bitmap */
1146	build_page_bitmap(p);
1147	}
1148
1149	/* we remove all the TBs in the range [start, end[ */
1150	/* XXX: see if in some cases it could be faster to invalidate all the code */
1151	tb = p->first_tb;
1152	while (tb != NULL) {
1153	n = (long)tb & 3;
1154	tb = (TranslationBlock *)((long)tb & ~3);
1155	tb_next = tb->page_next[n];
1156	/* NOTE: this is subtle as a TB may span two physical pages */
1157	if (n == 0) {
1158	/* NOTE: tb_end may be after the end of the page, but
1159	it is not a problem */
1160	tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1161	tb_end = tb_start + tb->size;
1162	} else {
1163	tb_start = tb->page_addr[1];
1164	tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1165	}
1166	if (!(tb_end <= start \|\| tb_start >= end)) {
1167	#ifdef TARGET_HAS_PRECISE_SMC
1168	if (current_tb_not_found) {
1169	current_tb_not_found = 0;
1170	current_tb = NULL;
1171	if (env->mem_io_pc) {
1172	/* now we have a real cpu fault */
1173	current_tb = tb_find_pc(env->mem_io_pc);
1174	}
1175	}
1176	if (current_tb == tb &&
1177	(current_tb->cflags & CF_COUNT_MASK) != 1) {
1178	/* If we are modifying the current TB, we must stop
1179	its execution. We could be more precise by checking
1180	that the modification is after the current PC, but it
1181	would require a specialized function to partially
1182	restore the CPU state */
1183
1184	current_tb_modified = 1;
1185	cpu_restore_state(current_tb, env,
1186	env->mem_io_pc, NULL);
1187	cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1188	&current_flags);
1189	}
1190	#endif /* TARGET_HAS_PRECISE_SMC */
1191	/* we need to do that to handle the case where a signal
1192	occurs while doing tb_phys_invalidate() */
1193	saved_tb = NULL;
1194	if (env) {
1195	saved_tb = env->current_tb;
1196	env->current_tb = NULL;
1197	}
1198	tb_phys_invalidate(tb, -1);
1199	if (env) {
1200	env->current_tb = saved_tb;
1201	if (env->interrupt_request && env->current_tb)
1202	cpu_interrupt(env, env->interrupt_request);
1203	}
1204	}
1205	tb = tb_next;
1206	}
1207	#if !defined(CONFIG_USER_ONLY)
1208	/* if no code remaining, no need to continue to use slow writes */
1209	if (!p->first_tb) {
1210	invalidate_page_bitmap(p);
1211	if (is_cpu_write_access) {
1212	tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1213	}
1214	}
1215	#endif
1216	#ifdef TARGET_HAS_PRECISE_SMC
1217	if (current_tb_modified) {
1218	/* we generate a block containing just the instruction
1219	modifying the memory. It will ensure that it cannot modify
1220	itself */
1221	env->current_tb = NULL;
1222	tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1223	cpu_resume_from_signal(env, NULL);
1224	}
1225	#endif
1226	}
1227
1228	/* len must be <= 8 and start must be a multiple of len */
1229	static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1230	{
1231	PageDesc *p;
1232	int offset, b;
1233	#if 0
1234	if (1) {
1235	qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1236	cpu_single_env->mem_io_vaddr, len,
1237	cpu_single_env->eip,
1238	cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1239	}
1240	#endif
1241	p = page_find(start >> TARGET_PAGE_BITS);
1242	if (!p)
1243	return;
1244	if (p->code_bitmap) {
1245	offset = start & ~TARGET_PAGE_MASK;
1246	b = p->code_bitmap[offset >> 3] >> (offset & 7);
1247	if (b & ((1 << len) - 1))
1248	goto do_invalidate;
1249	} else {
1250	do_invalidate:
1251	tb_invalidate_phys_page_range(start, start + len, 1);
1252	}
1253	}
1254
1255	#if !defined(CONFIG_SOFTMMU)
1256	static void tb_invalidate_phys_page(target_phys_addr_t addr,
1257	unsigned long pc, void *puc)
1258	{
1259	TranslationBlock *tb;
1260	PageDesc *p;
1261	int n;
1262	#ifdef TARGET_HAS_PRECISE_SMC
1263	TranslationBlock *current_tb = NULL;
1264	CPUState *env = cpu_single_env;
1265	int current_tb_modified = 0;
1266	target_ulong current_pc = 0;
1267	target_ulong current_cs_base = 0;
1268	int current_flags = 0;
1269	#endif
1270
1271	addr &= TARGET_PAGE_MASK;
1272	p = page_find(addr >> TARGET_PAGE_BITS);
1273	if (!p)
1274	return;
1275	tb = p->first_tb;
1276	#ifdef TARGET_HAS_PRECISE_SMC
1277	if (tb && pc != 0) {
1278	current_tb = tb_find_pc(pc);
1279	}
1280	#endif
1281	while (tb != NULL) {
1282	n = (long)tb & 3;
1283	tb = (TranslationBlock *)((long)tb & ~3);
1284	#ifdef TARGET_HAS_PRECISE_SMC
1285	if (current_tb == tb &&
1286	(current_tb->cflags & CF_COUNT_MASK) != 1) {
1287	/* If we are modifying the current TB, we must stop
1288	its execution. We could be more precise by checking
1289	that the modification is after the current PC, but it
1290	would require a specialized function to partially
1291	restore the CPU state */
1292
1293	current_tb_modified = 1;
1294	cpu_restore_state(current_tb, env, pc, puc);
1295	cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1296	&current_flags);
1297	}
1298	#endif /* TARGET_HAS_PRECISE_SMC */
1299	tb_phys_invalidate(tb, addr);
1300	tb = tb->page_next[n];
1301	}
1302	p->first_tb = NULL;
1303	#ifdef TARGET_HAS_PRECISE_SMC
1304	if (current_tb_modified) {
1305	/* we generate a block containing just the instruction
1306	modifying the memory. It will ensure that it cannot modify
1307	itself */
1308	env->current_tb = NULL;
1309	tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1310	cpu_resume_from_signal(env, puc);
1311	}
1312	#endif
1313	}
1314	#endif
1315
1316	/* add the tb in the target page and protect it if necessary */
1317	static inline void tb_alloc_page(TranslationBlock *tb,
1318	unsigned int n, target_ulong page_addr)
1319	{
1320	PageDesc *p;
1321	TranslationBlock *last_first_tb;
1322
1323	tb->page_addr[n] = page_addr;
1324	p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1325	tb->page_next[n] = p->first_tb;
1326	last_first_tb = p->first_tb;
1327	p->first_tb = (TranslationBlock *)((long)tb \| n);
1328	invalidate_page_bitmap(p);
1329
1330	#if defined(TARGET_HAS_SMC) \|\| 1
1331
1332	#if defined(CONFIG_USER_ONLY)
1333	if (p->flags & PAGE_WRITE) {
1334	target_ulong addr;
1335	PageDesc *p2;
1336	int prot;
1337
1338	/* force the host page as non writable (writes will have a
1339	page fault + mprotect overhead) */
1340	page_addr &= qemu_host_page_mask;
1341	prot = 0;
1342	for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1343	addr += TARGET_PAGE_SIZE) {
1344
1345	p2 = page_find (addr >> TARGET_PAGE_BITS);
1346	if (!p2)
1347	continue;
1348	prot \|= p2->flags;
1349	p2->flags &= ~PAGE_WRITE;
1350	page_get_flags(addr);
1351	}
1352	mprotect(g2h(page_addr), qemu_host_page_size,
1353	(prot & PAGE_BITS) & ~PAGE_WRITE);
1354	#ifdef DEBUG_TB_INVALIDATE
1355	printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1356	page_addr);
1357	#endif
1358	}
1359	#else
1360	/* if some code is already present, then the pages are already
1361	protected. So we handle the case where only the first TB is
1362	allocated in a physical page */
1363	if (!last_first_tb) {
1364	tlb_protect_code(page_addr);
1365	}
1366	#endif
1367
1368	#endif /* TARGET_HAS_SMC */
1369	}
1370
1371	/* Allocate a new translation block. Flush the translation buffer if
1372	too many translation blocks or too much generated code. */
1373	TranslationBlock *tb_alloc(target_ulong pc)
1374	{
1375	TranslationBlock *tb;
1376
1377	if (nb_tbs >= code_gen_max_blocks \|\|
1378	#ifndef VBOX
1379	(code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1380	#else
1381	(code_gen_ptr - code_gen_buffer) >= (int)code_gen_buffer_max_size)
1382	#endif
1383	return NULL;
1384	tb = &tbs[nb_tbs++];
1385	tb->pc = pc;
1386	tb->cflags = 0;
1387	return tb;
1388	}
1389
1390	void tb_free(TranslationBlock *tb)
1391	{
1392	/* In practice this is mostly used for single use temporary TB
1393	Ignore the hard cases and just back up if this TB happens to
1394	be the last one generated. */
1395	if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1396	code_gen_ptr = tb->tc_ptr;
1397	nb_tbs--;
1398	}
1399	}
1400
1401	/* add a new TB and link it to the physical page tables. phys_page2 is
1402	(-1) to indicate that only one page contains the TB. */
1403	void tb_link_phys(TranslationBlock *tb,
1404	target_ulong phys_pc, target_ulong phys_page2)
1405	{
1406	unsigned int h;
1407	TranslationBlock **ptb;
1408
1409	/* Grab the mmap lock to stop another thread invalidating this TB
1410	before we are done. */
1411	mmap_lock();
1412	/* add in the physical hash table */
1413	h = tb_phys_hash_func(phys_pc);
1414	ptb = &tb_phys_hash[h];
1415	tb->phys_hash_next = *ptb;
1416	*ptb = tb;
1417
1418	/* add in the page list */
1419	tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1420	if (phys_page2 != -1)
1421	tb_alloc_page(tb, 1, phys_page2);
1422	else
1423	tb->page_addr[1] = -1;
1424
1425	tb->jmp_first = (TranslationBlock *)((long)tb \| 2);
1426	tb->jmp_next[0] = NULL;
1427	tb->jmp_next[1] = NULL;
1428
1429	/* init original jump addresses */
1430	if (tb->tb_next_offset[0] != 0xffff)
1431	tb_reset_jump(tb, 0);
1432	if (tb->tb_next_offset[1] != 0xffff)
1433	tb_reset_jump(tb, 1);
1434
1435	#ifdef DEBUG_TB_CHECK
1436	tb_page_check();
1437	#endif
1438	mmap_unlock();
1439	}
1440
1441	/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1442	tb[1].tc_ptr. Return NULL if not found */
1443	TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1444	{
1445	int m_min, m_max, m;
1446	unsigned long v;
1447	TranslationBlock *tb;
1448
1449	if (nb_tbs <= 0)
1450	return NULL;
1451	if (tc_ptr < (unsigned long)code_gen_buffer \|\|
1452	tc_ptr >= (unsigned long)code_gen_ptr)
1453	return NULL;
1454	/* binary search (cf Knuth) */
1455	m_min = 0;
1456	m_max = nb_tbs - 1;
1457	while (m_min <= m_max) {
1458	m = (m_min + m_max) >> 1;
1459	tb = &tbs[m];
1460	v = (unsigned long)tb->tc_ptr;
1461	if (v == tc_ptr)
1462	return tb;
1463	else if (tc_ptr < v) {
1464	m_max = m - 1;
1465	} else {
1466	m_min = m + 1;
1467	}
1468	}
1469	return &tbs[m_max];
1470	}
1471
1472	static void tb_reset_jump_recursive(TranslationBlock *tb);
1473
1474	static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1475	{
1476	TranslationBlock tb1, tb_next, **ptb;
1477	unsigned int n1;
1478
1479	tb1 = tb->jmp_next[n];
1480	if (tb1 != NULL) {
1481	/* find head of list */
1482	for(;;) {
1483	n1 = (long)tb1 & 3;
1484	tb1 = (TranslationBlock *)((long)tb1 & ~3);
1485	if (n1 == 2)
1486	break;
1487	tb1 = tb1->jmp_next[n1];
1488	}
1489	/* we are now sure now that tb jumps to tb1 */
1490	tb_next = tb1;
1491
1492	/* remove tb from the jmp_first list */
1493	ptb = &tb_next->jmp_first;
1494	for(;;) {
1495	tb1 = *ptb;
1496	n1 = (long)tb1 & 3;
1497	tb1 = (TranslationBlock *)((long)tb1 & ~3);
1498	if (n1 == n && tb1 == tb)
1499	break;
1500	ptb = &tb1->jmp_next[n1];
1501	}
1502	*ptb = tb->jmp_next[n];
1503	tb->jmp_next[n] = NULL;
1504
1505	/* suppress the jump to next tb in generated code */
1506	tb_reset_jump(tb, n);
1507
1508	/* suppress jumps in the tb on which we could have jumped */
1509	tb_reset_jump_recursive(tb_next);
1510	}
1511	}
1512
1513	static void tb_reset_jump_recursive(TranslationBlock *tb)
1514	{
1515	tb_reset_jump_recursive2(tb, 0);
1516	tb_reset_jump_recursive2(tb, 1);
1517	}
1518
1519	#if defined(TARGET_HAS_ICE)
1520	static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1521	{
1522	target_phys_addr_t addr;
1523	target_ulong pd;
1524	ram_addr_t ram_addr;
1525	PhysPageDesc *p;
1526
1527	addr = cpu_get_phys_page_debug(env, pc);
1528	p = phys_page_find(addr >> TARGET_PAGE_BITS);
1529	if (!p) {
1530	pd = IO_MEM_UNASSIGNED;
1531	} else {
1532	pd = p->phys_offset;
1533	}
1534	ram_addr = (pd & TARGET_PAGE_MASK) \| (pc & ~TARGET_PAGE_MASK);
1535	tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1536	}
1537	#endif
1538
1539	/* Add a watchpoint. */
1540	int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1541	int flags, CPUWatchpoint **watchpoint)
1542	{
1543	target_ulong len_mask = ~(len - 1);
1544	CPUWatchpoint *wp;
1545
1546	/* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1547	if ((len != 1 && len != 2 && len != 4 && len != 8) \|\| (addr & ~len_mask)) {
1548	fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1549	TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1550	#ifndef VBOX
1551	return -EINVAL;
1552	#else
1553	return VERR_INVALID_PARAMETER;
1554	#endif
1555	}
1556	wp = qemu_malloc(sizeof(*wp));
1557
1558	wp->vaddr = addr;
1559	wp->len_mask = len_mask;
1560	wp->flags = flags;
1561
1562	/* keep all GDB-injected watchpoints in front */
1563	if (flags & BP_GDB)
1564	TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1565	else
1566	TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1567
1568	tlb_flush_page(env, addr);
1569
1570	if (watchpoint)
1571	*watchpoint = wp;
1572	return 0;
1573	}
1574
1575	/* Remove a specific watchpoint. */
1576	int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1577	int flags)
1578	{
1579	target_ulong len_mask = ~(len - 1);
1580	CPUWatchpoint *wp;
1581
1582	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1583	if (addr == wp->vaddr && len_mask == wp->len_mask
1584	&& flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1585	cpu_watchpoint_remove_by_ref(env, wp);
1586	return 0;
1587	}
1588	}
1589	#ifndef VBOX
1590	return -ENOENT;
1591	#else
1592	return VERR_NOT_FOUND;
1593	#endif
1594	}
1595
1596	/* Remove a specific watchpoint by reference. */
1597	void cpu_watchpoint_remove_by_ref(CPUState env, CPUWatchpoint watchpoint)
1598	{
1599	TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1600
1601	tlb_flush_page(env, watchpoint->vaddr);
1602
1603	qemu_free(watchpoint);
1604	}
1605
1606	/* Remove all matching watchpoints. */
1607	void cpu_watchpoint_remove_all(CPUState *env, int mask)
1608	{
1609	CPUWatchpoint wp, next;
1610
1611	TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1612	if (wp->flags & mask)
1613	cpu_watchpoint_remove_by_ref(env, wp);
1614	}
1615	}
1616
1617	/* Add a breakpoint. */
1618	int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1619	CPUBreakpoint **breakpoint)
1620	{
1621	#if defined(TARGET_HAS_ICE)
1622	CPUBreakpoint *bp;
1623
1624	bp = qemu_malloc(sizeof(*bp));
1625
1626	bp->pc = pc;
1627	bp->flags = flags;
1628
1629	/* keep all GDB-injected breakpoints in front */
1630	if (flags & BP_GDB)
1631	TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1632	else
1633	TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1634
1635	breakpoint_invalidate(env, pc);
1636
1637	if (breakpoint)
1638	*breakpoint = bp;
1639	return 0;
1640	#else
1641	return -ENOSYS;
1642	#endif
1643	}
1644
1645	/* Remove a specific breakpoint. */
1646	int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1647	{
1648	#if defined(TARGET_HAS_ICE)
1649	CPUBreakpoint *bp;
1650
1651	TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1652	if (bp->pc == pc && bp->flags == flags) {
1653	cpu_breakpoint_remove_by_ref(env, bp);
1654	return 0;
1655	}
1656	}
1657	# ifndef VBOX
1658	return -ENOENT;
1659	# else
1660	return VERR_NOT_FOUND;
1661	# endif
1662	#else
1663	return -ENOSYS;
1664	#endif
1665	}
1666
1667	/* Remove a specific breakpoint by reference. */
1668	void cpu_breakpoint_remove_by_ref(CPUState env, CPUBreakpoint breakpoint)
1669	{
1670	#if defined(TARGET_HAS_ICE)
1671	TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1672
1673	breakpoint_invalidate(env, breakpoint->pc);
1674
1675	qemu_free(breakpoint);
1676	#endif
1677	}
1678
1679	/* Remove all matching breakpoints. */
1680	void cpu_breakpoint_remove_all(CPUState *env, int mask)
1681	{
1682	#if defined(TARGET_HAS_ICE)
1683	CPUBreakpoint bp, next;
1684
1685	TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1686	if (bp->flags & mask)
1687	cpu_breakpoint_remove_by_ref(env, bp);
1688	}
1689	#endif
1690	}
1691
1692	/* enable or disable single step mode. EXCP_DEBUG is returned by the
1693	CPU loop after each instruction */
1694	void cpu_single_step(CPUState *env, int enabled)
1695	{
1696	#if defined(TARGET_HAS_ICE)
1697	if (env->singlestep_enabled != enabled) {
1698	env->singlestep_enabled = enabled;
1699	if (kvm_enabled())
1700	kvm_update_guest_debug(env, 0);
1701	else {
1702	/* must flush all the translated code to avoid inconsistencies */
1703	/* XXX: only flush what is necessary */
1704	tb_flush(env);
1705	}
1706	}
1707	#endif
1708	}
1709
1710	#ifndef VBOX
1711
1712	/* enable or disable low levels log */
1713	void cpu_set_log(int log_flags)
1714	{
1715	loglevel = log_flags;
1716	if (loglevel && !logfile) {
1717	logfile = fopen(logfilename, log_append ? "a" : "w");
1718	if (!logfile) {
1719	perror(logfilename);
1720	_exit(1);
1721	}
1722	#if !defined(CONFIG_SOFTMMU)
1723	/* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1724	{
1725	static char logfile_buf[4096];
1726	setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1727	}
1728	#else
1729	setvbuf(logfile, NULL, _IOLBF, 0);
1730	#endif
1731	log_append = 1;
1732	}
1733	if (!loglevel && logfile) {
1734	fclose(logfile);
1735	logfile = NULL;
1736	}
1737	}
1738
1739	void cpu_set_log_filename(const char *filename)
1740	{
1741	logfilename = strdup(filename);
1742	if (logfile) {
1743	fclose(logfile);
1744	logfile = NULL;
1745	}
1746	cpu_set_log(loglevel);
1747	}
1748
1749	#endif /* !VBOX */
1750
1751	static void cpu_unlink_tb(CPUState *env)
1752	{
1753	#if defined(USE_NPTL)
1754	/* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1755	problem and hope the cpu will stop of its own accord. For userspace
1756	emulation this often isn't actually as bad as it sounds. Often
1757	signals are used primarily to interrupt blocking syscalls. */
1758	#else
1759	TranslationBlock *tb;
1760	static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1761
1762	tb = env->current_tb;
1763	/* if the cpu is currently executing code, we must unlink it and
1764	all the potentially executing TB */
1765	if (tb && !testandset(&interrupt_lock)) {
1766	env->current_tb = NULL;
1767	tb_reset_jump_recursive(tb);
1768	resetlock(&interrupt_lock);
1769	}
1770	#endif
1771	}
1772
1773	/* mask must never be zero, except for A20 change call */
1774	void cpu_interrupt(CPUState *env, int mask)
1775	{
1776	int old_mask;
1777
1778	old_mask = env->interrupt_request;
1779	#ifndef VBOX
1780	env->interrupt_request \|= mask;
1781	#else /* VBOX */
1782	VM_ASSERT_EMT(env->pVM);
1783	ASMAtomicOrS32((int32_t volatile *)&env->interrupt_request, mask);
1784	#endif /* VBOX */
1785
1786	#ifndef VBOX
1787	#ifndef CONFIG_USER_ONLY
1788	/*
1789	* If called from iothread context, wake the target cpu in
1790	* case its halted.
1791	*/
1792	if (!qemu_cpu_self(env)) {
1793	qemu_cpu_kick(env);
1794	return;
1795	}
1796	#endif
1797	#endif /* !VBOX */
1798
1799	if (use_icount) {
1800	env->icount_decr.u16.high = 0xffff;
1801	#ifndef CONFIG_USER_ONLY
1802	if (!can_do_io(env)
1803	&& (mask & ~old_mask) != 0) {
1804	cpu_abort(env, "Raised interrupt while not in I/O function");
1805	}
1806	#endif
1807	} else {
1808	cpu_unlink_tb(env);
1809	}
1810	}
1811
1812	void cpu_reset_interrupt(CPUState *env, int mask)
1813	{
1814	#ifdef VBOX
1815	/*
1816	* Note: the current implementation can be executed by another thread without problems; make sure this remains true
1817	* for future changes!
1818	*/
1819	ASMAtomicAndS32((int32_t volatile *)&env->interrupt_request, ~mask);
1820	#else /* !VBOX */
1821	env->interrupt_request &= ~mask;
1822	#endif /* !VBOX */
1823	}
1824
1825	void cpu_exit(CPUState *env)
1826	{
1827	env->exit_request = 1;
1828	cpu_unlink_tb(env);
1829	}
1830
1831	#ifndef VBOX
1832	const CPULogItem cpu_log_items[] = {
1833	{ CPU_LOG_TB_OUT_ASM, "out_asm",
1834	"show generated host assembly code for each compiled TB" },
1835	{ CPU_LOG_TB_IN_ASM, "in_asm",
1836	"show target assembly code for each compiled TB" },
1837	{ CPU_LOG_TB_OP, "op",
1838	"show micro ops for each compiled TB" },
1839	{ CPU_LOG_TB_OP_OPT, "op_opt",
1840	"show micro ops "
1841	#ifdef TARGET_I386
1842	"before eflags optimization and "
1843	#endif
1844	"after liveness analysis" },
1845	{ CPU_LOG_INT, "int",
1846	"show interrupts/exceptions in short format" },
1847	{ CPU_LOG_EXEC, "exec",
1848	"show trace before each executed TB (lots of logs)" },
1849	{ CPU_LOG_TB_CPU, "cpu",
1850	"show CPU state before block translation" },
1851	#ifdef TARGET_I386
1852	{ CPU_LOG_PCALL, "pcall",
1853	"show protected mode far calls/returns/exceptions" },
1854	{ CPU_LOG_RESET, "cpu_reset",
1855	"show CPU state before CPU resets" },
1856	#endif
1857	#ifdef DEBUG_IOPORT
1858	{ CPU_LOG_IOPORT, "ioport",
1859	"show all i/o ports accesses" },
1860	#endif
1861	{ 0, NULL, NULL },
1862	};
1863
1864	static int cmp1(const char s1, int n, const char s2)
1865	{
1866	if (strlen(s2) != n)
1867	return 0;
1868	return memcmp(s1, s2, n) == 0;
1869	}
1870
1871	/* takes a comma separated list of log masks. Return 0 if error. */
1872	int cpu_str_to_log_mask(const char *str)
1873	{
1874	const CPULogItem *item;
1875	int mask;
1876	const char p, p1;
1877
1878	p = str;
1879	mask = 0;
1880	for(;;) {
1881	p1 = strchr(p, ',');
1882	if (!p1)
1883	p1 = p + strlen(p);
1884	if(cmp1(p,p1-p,"all")) {
1885	for(item = cpu_log_items; item->mask != 0; item++) {
1886	mask \|= item->mask;
1887	}
1888	} else {
1889	for(item = cpu_log_items; item->mask != 0; item++) {
1890	if (cmp1(p, p1 - p, item->name))
1891	goto found;
1892	}
1893	return 0;
1894	}
1895	found:
1896	mask \|= item->mask;
1897	if (*p1 != ',')
1898	break;
1899	p = p1 + 1;
1900	}
1901	return mask;
1902	}
1903	#endif /* !VBOX */
1904
1905	#ifndef VBOX /* VBOX: we have our own routine. */
1906	void cpu_abort(CPUState env, const char fmt, ...)
1907	{
1908	va_list ap;
1909	va_list ap2;
1910
1911	va_start(ap, fmt);
1912	va_copy(ap2, ap);
1913	fprintf(stderr, "qemu: fatal: ");
1914	vfprintf(stderr, fmt, ap);
1915	fprintf(stderr, "\n");
1916	#ifdef TARGET_I386
1917	cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU \| X86_DUMP_CCOP);
1918	#else
1919	cpu_dump_state(env, stderr, fprintf, 0);
1920	#endif
1921	if (qemu_log_enabled()) {
1922	qemu_log("qemu: fatal: ");
1923	qemu_log_vprintf(fmt, ap2);
1924	qemu_log("\n");
1925	#ifdef TARGET_I386
1926	log_cpu_state(env, X86_DUMP_FPU \| X86_DUMP_CCOP);
1927	#else
1928	log_cpu_state(env, 0);
1929	#endif
1930	qemu_log_flush();
1931	qemu_log_close();
1932	}
1933	va_end(ap2);
1934	va_end(ap);
1935	abort();
1936	}
1937	#endif /* !VBOX */
1938
1939	#ifndef VBOX
1940	CPUState cpu_copy(CPUState env)
1941	{
1942	CPUState *new_env = cpu_init(env->cpu_model_str);
1943	CPUState *next_cpu = new_env->next_cpu;
1944	int cpu_index = new_env->cpu_index;
1945	#if defined(TARGET_HAS_ICE)
1946	CPUBreakpoint *bp;
1947	CPUWatchpoint *wp;
1948	#endif
1949
1950	memcpy(new_env, env, sizeof(CPUState));
1951
1952	/* Preserve chaining and index. */
1953	new_env->next_cpu = next_cpu;
1954	new_env->cpu_index = cpu_index;
1955
1956	/* Clone all break/watchpoints.
1957	Note: Once we support ptrace with hw-debug register access, make sure
1958	BP_CPU break/watchpoints are handled correctly on clone. */
1959	TAILQ_INIT(&env->breakpoints);
1960	TAILQ_INIT(&env->watchpoints);
1961	#if defined(TARGET_HAS_ICE)
1962	TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1963	cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1964	}
1965	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1966	cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1967	wp->flags, NULL);
1968	}
1969	#endif
1970
1971	return new_env;
1972	}
1973	#endif /* !VBOX */
1974
1975	#if !defined(CONFIG_USER_ONLY)
1976
1977	static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1978	{
1979	unsigned int i;
1980
1981	/* Discard jump cache entries for any tb which might potentially
1982	overlap the flushed page. */
1983	i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1984	memset (&env->tb_jmp_cache[i], 0,
1985	TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1986
1987	i = tb_jmp_cache_hash_page(addr);
1988	memset (&env->tb_jmp_cache[i], 0,
1989	TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1990
1991	#ifdef VBOX
1992	/* inform raw mode about TLB page flush */
1993	remR3FlushPage(env, addr);
1994	#endif /* VBOX */
1995	}
1996
1997	static CPUTLBEntry s_cputlb_empty_entry = {
1998	.addr_read = -1,
1999	.addr_write = -1,
2000	.addr_code = -1,
2001	.addend = -1,
2002	};
2003
2004	/* NOTE: if flush_global is true, also flush global entries (not
2005	implemented yet) */
2006	void tlb_flush(CPUState *env, int flush_global)
2007	{
2008	int i;
2009
2010	#if defined(DEBUG_TLB)
2011	printf("tlb_flush:\n");
2012	#endif
2013	/* must reset current TB so that interrupts cannot modify the
2014	links while we are modifying them */
2015	env->current_tb = NULL;
2016
2017	for(i = 0; i < CPU_TLB_SIZE; i++) {
2018	int mmu_idx;
2019	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2020	env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
2021	}
2022	}
2023
2024	memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
2025
2026	#ifdef CONFIG_KQEMU
2027	if (env->kqemu_enabled) {
2028	kqemu_flush(env, flush_global);
2029	}
2030	#endif
2031	#ifdef VBOX
2032	/* inform raw mode about TLB flush */
2033	remR3FlushTLB(env, flush_global);
2034	#endif
2035	tlb_flush_count++;
2036	}
2037
2038	static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
2039	{
2040	if (addr == (tlb_entry->addr_read &
2041	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
2042	addr == (tlb_entry->addr_write &
2043	(TARGET_PAGE_MASK \| TLB_INVALID_MASK)) \|\|
2044	addr == (tlb_entry->addr_code &
2045	(TARGET_PAGE_MASK \| TLB_INVALID_MASK))) {
2046	*tlb_entry = s_cputlb_empty_entry;
2047	}
2048	}
2049
2050	void tlb_flush_page(CPUState *env, target_ulong addr)
2051	{
2052	int i;
2053	int mmu_idx;
2054
2055	#if defined(DEBUG_TLB)
2056	printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
2057	#endif
2058	/* must reset current TB so that interrupts cannot modify the
2059	links while we are modifying them */
2060	env->current_tb = NULL;
2061
2062	addr &= TARGET_PAGE_MASK;
2063	i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2064	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2065	tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
2066
2067	tlb_flush_jmp_cache(env, addr);
2068
2069	#ifdef CONFIG_KQEMU
2070	if (env->kqemu_enabled) {
2071	kqemu_flush_page(env, addr);
2072	}
2073	#endif
2074	}
2075
2076	/* update the TLBs so that writes to code in the virtual page 'addr'
2077	can be detected */
2078	static void tlb_protect_code(ram_addr_t ram_addr)
2079	{
2080	cpu_physical_memory_reset_dirty(ram_addr,
2081	ram_addr + TARGET_PAGE_SIZE,
2082	CODE_DIRTY_FLAG);
2083	#if defined(VBOX) && defined(REM_MONITOR_CODE_PAGES)
2084	/** @todo Retest this? This function has changed... */
2085	remR3ProtectCode(cpu_single_env, ram_addr);
2086	#endif
2087	}
2088
2089	/* update the TLB so that writes in physical page 'phys_addr' are no longer
2090	tested for self modifying code */
2091	static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2092	target_ulong vaddr)
2093	{
2094	#ifdef VBOX
2095	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
2096	#endif
2097	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] \|= CODE_DIRTY_FLAG;
2098	}
2099
2100	static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2101	unsigned long start, unsigned long length)
2102	{
2103	unsigned long addr;
2104
2105	#ifdef VBOX
2106	if (start & 3)
2107	return;
2108	#endif
2109	if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2110	addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2111	if ((addr - start) < length) {
2112	tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) \| TLB_NOTDIRTY;
2113	}
2114	}
2115	}
2116
2117	/* Note: start and end must be within the same ram block. */
2118	void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2119	int dirty_flags)
2120	{
2121	CPUState *env;
2122	unsigned long length, start1;
2123	int i, mask, len;
2124	uint8_t *p;
2125
2126	start &= TARGET_PAGE_MASK;
2127	end = TARGET_PAGE_ALIGN(end);
2128
2129	length = end - start;
2130	if (length == 0)
2131	return;
2132	len = length >> TARGET_PAGE_BITS;
2133	#ifdef CONFIG_KQEMU
2134	/* XXX: should not depend on cpu context */
2135	env = first_cpu;
2136	if (env->kqemu_enabled) {
2137	ram_addr_t addr;
2138	addr = start;
2139	for(i = 0; i < len; i++) {
2140	kqemu_set_notdirty(env, addr);
2141	addr += TARGET_PAGE_SIZE;
2142	}
2143	}
2144	#endif
2145	mask = ~dirty_flags;
2146	p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
2147	#ifdef VBOX
2148	if (RT_LIKELY((start >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
2149	#endif
2150	for(i = 0; i < len; i++)
2151	p[i] &= mask;
2152
2153	/* we modify the TLB cache so that the dirty bit will be set again
2154	when accessing the range */
2155	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2156	start1 = start;
2157	#elif !defined(VBOX)
2158	start1 = (unsigned long)qemu_get_ram_ptr(start);
2159	/* Chek that we don't span multiple blocks - this breaks the
2160	address comparisons below. */
2161	if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2162	!= (end - 1) - start) {
2163	abort();
2164	}
2165	#else
2166	start1 = (unsigned long)remR3TlbGCPhys2Ptr(first_cpu, start, 1 /fWritable/); /** @todo page replacing (sharing or read only) may cause trouble, fix interface/whatever. */
2167	#endif
2168
2169	for(env = first_cpu; env != NULL; env = env->next_cpu) {
2170	int mmu_idx;
2171	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2172	for(i = 0; i < CPU_TLB_SIZE; i++)
2173	tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2174	start1, length);
2175	}
2176	}
2177	}
2178
2179	#ifndef VBOX
2180	int cpu_physical_memory_set_dirty_tracking(int enable)
2181	{
2182	in_migration = enable;
2183	if (kvm_enabled()) {
2184	return kvm_set_migration_log(enable);
2185	}
2186	return 0;
2187	}
2188
2189	int cpu_physical_memory_get_dirty_tracking(void)
2190	{
2191	return in_migration;
2192	}
2193	#endif /* !VBOX */
2194
2195	int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2196	target_phys_addr_t end_addr)
2197	{
2198	#ifndef VBOX
2199	int ret = 0;
2200
2201	if (kvm_enabled())
2202	ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
2203	return ret;
2204	#else
2205	return 0;
2206	#endif
2207	}
2208
2209	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2210	DECLINLINE(void) tlb_update_dirty(CPUTLBEntry *tlb_entry, target_phys_addr_t phys_addend)
2211	#else
2212	static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2213	#endif
2214	{
2215	ram_addr_t ram_addr;
2216	void *p;
2217
2218	if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2219	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2220	ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2221	#elif !defined(VBOX)
2222	p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2223	+ tlb_entry->addend);
2224	ram_addr = qemu_ram_addr_from_host(p);
2225	#else
2226	Assert(phys_addend != -1);
2227	ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + phys_addend;
2228	#endif
2229	if (!cpu_physical_memory_is_dirty(ram_addr)) {
2230	tlb_entry->addr_write \|= TLB_NOTDIRTY;
2231	}
2232	}
2233	}
2234
2235	/* update the TLB according to the current state of the dirty bits */
2236	void cpu_tlb_update_dirty(CPUState *env)
2237	{
2238	int i;
2239	int mmu_idx;
2240	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2241	for(i = 0; i < CPU_TLB_SIZE; i++)
2242	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2243	tlb_update_dirty(&env->tlb_table[mmu_idx][i], env->phys_addends[mmu_idx][i]);
2244	#else
2245	tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2246	#endif
2247	}
2248	}
2249
2250	static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2251	{
2252	if (tlb_entry->addr_write == (vaddr \| TLB_NOTDIRTY))
2253	tlb_entry->addr_write = vaddr;
2254	}
2255
2256	/* update the TLB corresponding to virtual page vaddr
2257	so that it is no longer dirty */
2258	static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2259	{
2260	int i;
2261	int mmu_idx;
2262
2263	vaddr &= TARGET_PAGE_MASK;
2264	i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2265	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2266	tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2267	}
2268
2269	/* add a new TLB entry. At most one entry for a given virtual address
2270	is permitted. Return 0 if OK or 2 if the page could not be mapped
2271	(can only happen in non SOFTMMU mode for I/O pages or pages
2272	conflicting with the host address space). */
2273	int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2274	target_phys_addr_t paddr, int prot,
2275	int mmu_idx, int is_softmmu)
2276	{
2277	PhysPageDesc *p;
2278	unsigned long pd;
2279	unsigned int index;
2280	target_ulong address;
2281	target_ulong code_address;
2282	target_phys_addr_t addend;
2283	int ret;
2284	CPUTLBEntry *te;
2285	CPUWatchpoint *wp;
2286	target_phys_addr_t iotlb;
2287	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2288	int read_mods = 0, write_mods = 0, code_mods = 0;
2289	#endif
2290
2291	p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2292	if (!p) {
2293	pd = IO_MEM_UNASSIGNED;
2294	} else {
2295	pd = p->phys_offset;
2296	}
2297	#if defined(DEBUG_TLB)
2298	printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2299	vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2300	#endif
2301
2302	ret = 0;
2303	address = vaddr;
2304	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2305	/* IO memory case (romd handled later) */
2306	address \|= TLB_MMIO;
2307	}
2308	#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
2309	addend = pd & TARGET_PAGE_MASK;
2310	#elif !defined(VBOX)
2311	addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2312	#else
2313	/** @todo this is racing the phys_page_find call above since it may register
2314	* a new chunk of memory... */
2315	addend = (unsigned long)remR3TlbGCPhys2Ptr(env, pd & TARGET_PAGE_MASK, !!(prot & PAGE_WRITE));
2316	#endif
2317
2318	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2319	/* Normal RAM. */
2320	iotlb = pd & TARGET_PAGE_MASK;
2321	if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2322	iotlb \|= IO_MEM_NOTDIRTY;
2323	else
2324	iotlb \|= IO_MEM_ROM;
2325	} else {
2326	/* IO handlers are currently passed a physical address.
2327	It would be nice to pass an offset from the base address
2328	of that region. This would avoid having to special case RAM,
2329	and avoid full address decoding in every device.
2330	We can't use the high bits of pd for this because
2331	IO_MEM_ROMD uses these as a ram address. */
2332	iotlb = (pd & ~TARGET_PAGE_MASK);
2333	#ifndef VBOX
2334	if (p) {
2335	#else
2336	if ( p->phys_offset
2337	&& (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iMMIOMemType
2338	&& (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iHandlerMemType) {
2339	#endif
2340	iotlb += p->region_offset;
2341	} else {
2342	iotlb += paddr;
2343	}
2344	}
2345
2346	code_address = address;
2347	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2348
2349	if (addend & 0x3)
2350	{
2351	if (addend & 0x2)
2352	{
2353	/* catch write */
2354	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM)
2355	write_mods \|= TLB_MMIO;
2356	}
2357	else if (addend & 0x1)
2358	{
2359	/* catch all */
2360	if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM)
2361	{
2362	read_mods \|= TLB_MMIO;
2363	write_mods \|= TLB_MMIO;
2364	code_mods \|= TLB_MMIO;
2365	}
2366	}
2367	if ((iotlb & ~TARGET_PAGE_MASK) == 0)
2368	iotlb = env->pVM->rem.s.iHandlerMemType + paddr;
2369	addend &= ~(target_ulong)0x3;
2370	}
2371
2372	#endif
2373	/* Make accesses to pages with watchpoints go via the
2374	watchpoint trap routines. */
2375	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2376	if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2377	iotlb = io_mem_watch + paddr;
2378	/* TODO: The memory case can be optimized by not trapping
2379	reads of pages with a write breakpoint. */
2380	address \|= TLB_MMIO;
2381	}
2382	}
2383
2384	index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2385	env->iotlb[mmu_idx][index] = iotlb - vaddr;
2386	te = &env->tlb_table[mmu_idx][index];
2387	te->addend = addend - vaddr;
2388	if (prot & PAGE_READ) {
2389	te->addr_read = address;
2390	} else {
2391	te->addr_read = -1;
2392	}
2393
2394	if (prot & PAGE_EXEC) {
2395	te->addr_code = code_address;
2396	} else {
2397	te->addr_code = -1;
2398	}
2399	if (prot & PAGE_WRITE) {
2400	if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM \|\|
2401	(pd & IO_MEM_ROMD)) {
2402	/* Write access calls the I/O callback. */
2403	te->addr_write = address \| TLB_MMIO;
2404	} else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2405	!cpu_physical_memory_is_dirty(pd)) {
2406	te->addr_write = address \| TLB_NOTDIRTY;
2407	} else {
2408	te->addr_write = address;
2409	}
2410	} else {
2411	te->addr_write = -1;
2412	}
2413
2414	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
2415	if (prot & PAGE_READ)
2416	te->addr_read \|= read_mods;
2417	if (prot & PAGE_EXEC)
2418	te->addr_code \|= code_mods;
2419	if (prot & PAGE_WRITE)
2420	te->addr_write \|= write_mods;
2421
2422	env->phys_addends[mmu_idx][index] = (pd & TARGET_PAGE_MASK)- vaddr;
2423	#endif
2424
2425	#ifdef VBOX
2426	/* inform raw mode about TLB page change */
2427	remR3FlushPage(env, vaddr);
2428	#endif
2429	return ret;
2430	}
2431
2432	#else
2433
2434	void tlb_flush(CPUState *env, int flush_global)
2435	{
2436	}
2437
2438	void tlb_flush_page(CPUState *env, target_ulong addr)
2439	{
2440	}
2441
2442	int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2443	target_phys_addr_t paddr, int prot,
2444	int mmu_idx, int is_softmmu)
2445	{
2446	return 0;
2447	}
2448
2449	#ifndef VBOX
2450
2451	/*
2452	* Walks guest process memory "regions" one by one
2453	* and calls callback function 'fn' for each region.
2454	*/
2455	int walk_memory_regions(void *priv,
2456	int (fn)(void , unsigned long, unsigned long, unsigned long))
2457	{
2458	unsigned long start, end;
2459	PageDesc *p = NULL;
2460	int i, j, prot, prot1;
2461	int rc = 0;
2462
2463	start = end = -1;
2464	prot = 0;
2465
2466	for (i = 0; i <= L1_SIZE; i++) {
2467	p = (i < L1_SIZE) ? l1_map[i] : NULL;
2468	for (j = 0; j < L2_SIZE; j++) {
2469	prot1 = (p == NULL) ? 0 : p[j].flags;
2470	/*
2471	* "region" is one continuous chunk of memory
2472	* that has same protection flags set.
2473	*/
2474	if (prot1 != prot) {
2475	end = (i << (32 - L1_BITS)) \| (j << TARGET_PAGE_BITS);
2476	if (start != -1) {
2477	rc = (*fn)(priv, start, end, prot);
2478	/* callback can stop iteration by returning != 0 */
2479	if (rc != 0)
2480	return (rc);
2481	}
2482	if (prot1 != 0)
2483	start = end;
2484	else
2485	start = -1;
2486	prot = prot1;
2487	}
2488	if (p == NULL)
2489	break;
2490	}
2491	}
2492	return (rc);
2493	}
2494
2495	static int dump_region(void *priv, unsigned long start,
2496	unsigned long end, unsigned long prot)
2497	{
2498	FILE f = (FILE )priv;
2499
2500	(void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2501	start, end, end - start,
2502	((prot & PAGE_READ) ? 'r' : '-'),
2503	((prot & PAGE_WRITE) ? 'w' : '-'),
2504	((prot & PAGE_EXEC) ? 'x' : '-'));
2505
2506	return (0);
2507	}
2508
2509	/* dump memory mappings */
2510	void page_dump(FILE *f)
2511	{
2512	(void) fprintf(f, "%-8s %-8s %-8s %s\n",
2513	"start", "end", "size", "prot");
2514	walk_memory_regions(f, dump_region);
2515	}
2516
2517	#endif /* !VBOX */
2518
2519	int page_get_flags(target_ulong address)
2520	{
2521	PageDesc *p;
2522
2523	p = page_find(address >> TARGET_PAGE_BITS);
2524	if (!p)
2525	return 0;
2526	return p->flags;
2527	}
2528
2529	/* modify the flags of a page and invalidate the code if
2530	necessary. The flag PAGE_WRITE_ORG is positioned automatically
2531	depending on PAGE_WRITE */
2532	void page_set_flags(target_ulong start, target_ulong end, int flags)
2533	{
2534	PageDesc *p;
2535	target_ulong addr;
2536
2537	/* mmap_lock should already be held. */
2538	start = start & TARGET_PAGE_MASK;
2539	end = TARGET_PAGE_ALIGN(end);
2540	if (flags & PAGE_WRITE)
2541	flags \|= PAGE_WRITE_ORG;
2542	#ifdef VBOX
2543	AssertMsgFailed(("We shouldn't be here, and if we should, we must have an env to do the proper locking!\n"));
2544	#endif
2545	for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2546	p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2547	/* We may be called for host regions that are outside guest
2548	address space. */
2549	if (!p)
2550	return;
2551	/* if the write protection is set, then we invalidate the code
2552	inside */
2553	if (!(p->flags & PAGE_WRITE) &&
2554	(flags & PAGE_WRITE) &&
2555	p->first_tb) {
2556	tb_invalidate_phys_page(addr, 0, NULL);
2557	}
2558	p->flags = flags;
2559	}
2560	}
2561
2562	int page_check_range(target_ulong start, target_ulong len, int flags)
2563	{
2564	PageDesc *p;
2565	target_ulong end;
2566	target_ulong addr;
2567
2568	if (start + len < start)
2569	/* we've wrapped around */
2570	return -1;
2571
2572	end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2573	start = start & TARGET_PAGE_MASK;
2574
2575	for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2576	p = page_find(addr >> TARGET_PAGE_BITS);
2577	if( !p )
2578	return -1;
2579	if( !(p->flags & PAGE_VALID) )
2580	return -1;
2581
2582	if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2583	return -1;
2584	if (flags & PAGE_WRITE) {
2585	if (!(p->flags & PAGE_WRITE_ORG))
2586	return -1;
2587	/* unprotect the page if it was put read-only because it
2588	contains translated code */
2589	if (!(p->flags & PAGE_WRITE)) {
2590	if (!page_unprotect(addr, 0, NULL))
2591	return -1;
2592	}
2593	return 0;
2594	}
2595	}
2596	return 0;
2597	}
2598
2599	/* called from signal handler: invalidate the code and unprotect the
2600	page. Return TRUE if the fault was successfully handled. */
2601	int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2602	{
2603	unsigned int page_index, prot, pindex;
2604	PageDesc p, p1;
2605	target_ulong host_start, host_end, addr;
2606
2607	/* Technically this isn't safe inside a signal handler. However we
2608	know this only ever happens in a synchronous SEGV handler, so in
2609	practice it seems to be ok. */
2610	mmap_lock();
2611
2612	host_start = address & qemu_host_page_mask;
2613	page_index = host_start >> TARGET_PAGE_BITS;
2614	p1 = page_find(page_index);
2615	if (!p1) {
2616	mmap_unlock();
2617	return 0;
2618	}
2619	host_end = host_start + qemu_host_page_size;
2620	p = p1;
2621	prot = 0;
2622	for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2623	prot \|= p->flags;
2624	p++;
2625	}
2626	/* if the page was really writable, then we change its
2627	protection back to writable */
2628	if (prot & PAGE_WRITE_ORG) {
2629	pindex = (address - host_start) >> TARGET_PAGE_BITS;
2630	if (!(p1[pindex].flags & PAGE_WRITE)) {
2631	mprotect((void *)g2h(host_start), qemu_host_page_size,
2632	(prot & PAGE_BITS) \| PAGE_WRITE);
2633	p1[pindex].flags \|= PAGE_WRITE;
2634	/* and since the content will be modified, we must invalidate
2635	the corresponding translated code. */
2636	tb_invalidate_phys_page(address, pc, puc);
2637	#ifdef DEBUG_TB_CHECK
2638	tb_invalidate_check(address);
2639	#endif
2640	mmap_unlock();
2641	return 1;
2642	}
2643	}
2644	mmap_unlock();
2645	return 0;
2646	}
2647
2648	static inline void tlb_set_dirty(CPUState *env,
2649	unsigned long addr, target_ulong vaddr)
2650	{
2651	}
2652	#endif /* defined(CONFIG_USER_ONLY) */
2653
2654	#if !defined(CONFIG_USER_ONLY)
2655
2656	static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2657	ram_addr_t memory, ram_addr_t region_offset);
2658	static void subpage_init (target_phys_addr_t base, ram_addr_t phys,
2659	ram_addr_t orig_memory, ram_addr_t region_offset);
2660	#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2661	need_subpage) \
2662	do { \
2663	if (addr > start_addr) \
2664	start_addr2 = 0; \
2665	else { \
2666	start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2667	if (start_addr2 > 0) \
2668	need_subpage = 1; \
2669	} \
2670	\
2671	if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2672	end_addr2 = TARGET_PAGE_SIZE - 1; \
2673	else { \
2674	end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2675	if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2676	need_subpage = 1; \
2677	} \
2678	} while (0)
2679
2680	/* register physical memory. 'size' must be a multiple of the target
2681	page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2682	io memory page. The address used when calling the IO function is
2683	the offset from the start of the region, plus region_offset. Both
2684	start_addr and region_offset are rounded down to a page boundary
2685	before calculating this offset. This should not be a problem unless
2686	the low bits of start_addr and region_offset differ. */
2687	void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2688	ram_addr_t size,
2689	ram_addr_t phys_offset,
2690	ram_addr_t region_offset)
2691	{
2692	target_phys_addr_t addr, end_addr;
2693	PhysPageDesc *p;
2694	CPUState *env;
2695	ram_addr_t orig_size = size;
2696	void *subpage;
2697
2698	#ifdef CONFIG_KQEMU
2699	/* XXX: should not depend on cpu context */
2700	env = first_cpu;
2701	if (env->kqemu_enabled) {
2702	kqemu_set_phys_mem(start_addr, size, phys_offset);
2703	}
2704	#endif
2705	if (kvm_enabled())
2706	kvm_set_phys_mem(start_addr, size, phys_offset);
2707
2708	if (phys_offset == IO_MEM_UNASSIGNED) {
2709	region_offset = start_addr;
2710	}
2711	region_offset &= TARGET_PAGE_MASK;
2712	size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2713	end_addr = start_addr + (target_phys_addr_t)size;
2714	for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2715	p = phys_page_find(addr >> TARGET_PAGE_BITS);
2716	if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2717	ram_addr_t orig_memory = p->phys_offset;
2718	target_phys_addr_t start_addr2, end_addr2;
2719	int need_subpage = 0;
2720
2721	CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2722	need_subpage);
2723	if (need_subpage \|\| phys_offset & IO_MEM_SUBWIDTH) {
2724	if (!(orig_memory & IO_MEM_SUBPAGE)) {
2725	subpage = subpage_init((addr & TARGET_PAGE_MASK),
2726	&p->phys_offset, orig_memory,
2727	p->region_offset);
2728	} else {
2729	subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2730	>> IO_MEM_SHIFT];
2731	}
2732	subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2733	region_offset);
2734	p->region_offset = 0;
2735	} else {
2736	p->phys_offset = phys_offset;
2737	if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM \|\|
2738	(phys_offset & IO_MEM_ROMD))
2739	phys_offset += TARGET_PAGE_SIZE;
2740	}
2741	} else {
2742	p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2743	p->phys_offset = phys_offset;
2744	p->region_offset = region_offset;
2745	if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM \|\|
2746	(phys_offset & IO_MEM_ROMD)) {
2747	phys_offset += TARGET_PAGE_SIZE;
2748	} else {
2749	target_phys_addr_t start_addr2, end_addr2;
2750	int need_subpage = 0;
2751
2752	CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2753	end_addr2, need_subpage);
2754
2755	if (need_subpage \|\| phys_offset & IO_MEM_SUBWIDTH) {
2756	subpage = subpage_init((addr & TARGET_PAGE_MASK),
2757	&p->phys_offset, IO_MEM_UNASSIGNED,
2758	addr & TARGET_PAGE_MASK);
2759	subpage_register(subpage, start_addr2, end_addr2,
2760	phys_offset, region_offset);
2761	p->region_offset = 0;
2762	}
2763	}
2764	}
2765	region_offset += TARGET_PAGE_SIZE;
2766	}
2767
2768	/* since each CPU stores ram addresses in its TLB cache, we must
2769	reset the modified entries */
2770	/* XXX: slow ! */
2771	for(env = first_cpu; env != NULL; env = env->next_cpu) {
2772	tlb_flush(env, 1);
2773	}
2774	}
2775
2776	/* XXX: temporary until new memory mapping API */
2777	ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2778	{
2779	PhysPageDesc *p;
2780
2781	p = phys_page_find(addr >> TARGET_PAGE_BITS);
2782	if (!p)
2783	return IO_MEM_UNASSIGNED;
2784	return p->phys_offset;
2785	}
2786
2787	#ifndef VBOX
2788	void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2789	{
2790	if (kvm_enabled())
2791	kvm_coalesce_mmio_region(addr, size);
2792	}
2793
2794	void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2795	{
2796	if (kvm_enabled())
2797	kvm_uncoalesce_mmio_region(addr, size);
2798	}
2799
2800	#ifdef CONFIG_KQEMU
2801	/* XXX: better than nothing */
2802	static ram_addr_t kqemu_ram_alloc(ram_addr_t size)
2803	{
2804	ram_addr_t addr;
2805	if ((last_ram_offset + size) > kqemu_phys_ram_size) {
2806	fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2807	(uint64_t)size, (uint64_t)kqemu_phys_ram_size);
2808	abort();
2809	}
2810	addr = last_ram_offset;
2811	last_ram_offset = TARGET_PAGE_ALIGN(last_ram_offset + size);
2812	return addr;
2813	}
2814	#endif
2815
2816	ram_addr_t qemu_ram_alloc(ram_addr_t size)
2817	{
2818	RAMBlock *new_block;
2819
2820	#ifdef CONFIG_KQEMU
2821	if (kqemu_phys_ram_base) {
2822	return kqemu_ram_alloc(size);
2823	}
2824	#endif
2825
2826	size = TARGET_PAGE_ALIGN(size);
2827	new_block = qemu_malloc(sizeof(*new_block));
2828
2829	new_block->host = qemu_vmalloc(size);
2830	new_block->offset = last_ram_offset;
2831	new_block->length = size;
2832
2833	new_block->next = ram_blocks;
2834	ram_blocks = new_block;
2835
2836	phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2837	(last_ram_offset + size) >> TARGET_PAGE_BITS);
2838	memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2839	0xff, size >> TARGET_PAGE_BITS);
2840
2841	last_ram_offset += size;
2842
2843	if (kvm_enabled())
2844	kvm_setup_guest_memory(new_block->host, size);
2845
2846	return new_block->offset;
2847	}
2848
2849	void qemu_ram_free(ram_addr_t addr)
2850	{
2851	/* TODO: implement this. */
2852	}
2853
2854	/* Return a host pointer to ram allocated with qemu_ram_alloc.
2855	With the exception of the softmmu code in this file, this should
2856	only be used for local memory (e.g. video ram) that the device owns,
2857	and knows it isn't going to access beyond the end of the block.
2858
2859	It should not be used for general purpose DMA.
2860	Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2861	*/
2862	void *qemu_get_ram_ptr(ram_addr_t addr)
2863	{
2864	RAMBlock *prev;
2865	RAMBlock **prevp;
2866	RAMBlock *block;
2867
2868	#ifdef CONFIG_KQEMU
2869	if (kqemu_phys_ram_base) {
2870	return kqemu_phys_ram_base + addr;
2871	}
2872	#endif
2873
2874	prev = NULL;
2875	prevp = &ram_blocks;
2876	block = ram_blocks;
2877	while (block && (block->offset > addr
2878	\|\| block->offset + block->length <= addr)) {
2879	if (prev)
2880	prevp = &prev->next;
2881	prev = block;
2882	block = block->next;
2883	}
2884	if (!block) {
2885	fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2886	abort();
2887	}
2888	/* Move this entry to to start of the list. */
2889	if (prev) {
2890	prev->next = block->next;
2891	block->next = *prevp;
2892	*prevp = block;
2893	}
2894	return block->host + (addr - block->offset);
2895	}
2896
2897	/* Some of the softmmu routines need to translate from a host pointer
2898	(typically a TLB entry) back to a ram offset. */
2899	ram_addr_t qemu_ram_addr_from_host(void *ptr)
2900	{
2901	RAMBlock *prev;
2902	RAMBlock **prevp;
2903	RAMBlock *block;
2904	uint8_t *host = ptr;
2905
2906	#ifdef CONFIG_KQEMU
2907	if (kqemu_phys_ram_base) {
2908	return host - kqemu_phys_ram_base;
2909	}
2910	#endif
2911
2912	prev = NULL;
2913	prevp = &ram_blocks;
2914	block = ram_blocks;
2915	while (block && (block->host > host
2916	\|\| block->host + block->length <= host)) {
2917	if (prev)
2918	prevp = &prev->next;
2919	prev = block;
2920	block = block->next;
2921	}
2922	if (!block) {
2923	fprintf(stderr, "Bad ram pointer %p\n", ptr);
2924	abort();
2925	}
2926	return block->offset + (host - block->host);
2927	}
2928
2929	#endif /* !VBOX */
2930
2931	static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2932	{
2933	#ifdef DEBUG_UNASSIGNED
2934	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2935	#endif
2936	#if defined(TARGET_SPARC)
2937	do_unassigned_access(addr, 0, 0, 0, 1);
2938	#endif
2939	return 0;
2940	}
2941
2942	static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2943	{
2944	#ifdef DEBUG_UNASSIGNED
2945	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2946	#endif
2947	#if defined(TARGET_SPARC)
2948	do_unassigned_access(addr, 0, 0, 0, 2);
2949	#endif
2950	return 0;
2951	}
2952
2953	static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2954	{
2955	#ifdef DEBUG_UNASSIGNED
2956	printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2957	#endif
2958	#if defined(TARGET_SPARC)
2959	do_unassigned_access(addr, 0, 0, 0, 4);
2960	#endif
2961	return 0;
2962	}
2963
2964	static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2965	{
2966	#ifdef DEBUG_UNASSIGNED
2967	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2968	#endif
2969	#if defined(TARGET_SPARC)
2970	do_unassigned_access(addr, 1, 0, 0, 1);
2971	#endif
2972	}
2973
2974	static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2975	{
2976	#ifdef DEBUG_UNASSIGNED
2977	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2978	#endif
2979	#if defined(TARGET_SPARC)
2980	do_unassigned_access(addr, 1, 0, 0, 2);
2981	#endif
2982	}
2983
2984	static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2985	{
2986	#ifdef DEBUG_UNASSIGNED
2987	printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2988	#endif
2989	#if defined(TARGET_SPARC)
2990	do_unassigned_access(addr, 1, 0, 0, 4);
2991	#endif
2992	}
2993
2994	static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2995	unassigned_mem_readb,
2996	unassigned_mem_readw,
2997	unassigned_mem_readl,
2998	};
2999
3000	static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
3001	unassigned_mem_writeb,
3002	unassigned_mem_writew,
3003	unassigned_mem_writel,
3004	};
3005
3006	static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3007	uint32_t val)
3008	{
3009	int dirty_flags;
3010	#ifdef VBOX
3011	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3012	dirty_flags = 0xff;
3013	else
3014	#endif /* VBOX */
3015	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3016	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3017	#if !defined(CONFIG_USER_ONLY)
3018	tb_invalidate_phys_page_fast(ram_addr, 1);
3019	# ifdef VBOX
3020	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3021	dirty_flags = 0xff;
3022	else
3023	# endif /* VBOX */
3024	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3025	#endif
3026	}
3027	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3028	remR3PhysWriteU8(ram_addr, val);
3029	#else
3030	stb_p(qemu_get_ram_ptr(ram_addr), val);
3031	#endif
3032	#ifdef CONFIG_KQEMU
3033	if (cpu_single_env->kqemu_enabled &&
3034	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3035	kqemu_modify_page(cpu_single_env, ram_addr);
3036	#endif
3037	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3038	#ifdef VBOX
3039	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3040	#endif /* !VBOX */
3041	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3042	/* we remove the notdirty callback only if the code has been
3043	flushed */
3044	if (dirty_flags == 0xff)
3045	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3046	}
3047
3048	static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3049	uint32_t val)
3050	{
3051	int dirty_flags;
3052	#ifdef VBOX
3053	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3054	dirty_flags = 0xff;
3055	else
3056	#endif /* VBOX */
3057	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3058	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3059	#if !defined(CONFIG_USER_ONLY)
3060	tb_invalidate_phys_page_fast(ram_addr, 2);
3061	# ifdef VBOX
3062	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3063	dirty_flags = 0xff;
3064	else
3065	# endif /* VBOX */
3066	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3067	#endif
3068	}
3069	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3070	remR3PhysWriteU16(ram_addr, val);
3071	#else
3072	stw_p(qemu_get_ram_ptr(ram_addr), val);
3073	#endif
3074	#ifdef CONFIG_KQEMU
3075	if (cpu_single_env->kqemu_enabled &&
3076	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3077	kqemu_modify_page(cpu_single_env, ram_addr);
3078	#endif
3079	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3080	#ifdef VBOX
3081	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3082	#endif
3083	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3084	/* we remove the notdirty callback only if the code has been
3085	flushed */
3086	if (dirty_flags == 0xff)
3087	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3088	}
3089
3090	static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3091	uint32_t val)
3092	{
3093	int dirty_flags;
3094	#ifdef VBOX
3095	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3096	dirty_flags = 0xff;
3097	else
3098	#endif /* VBOX */
3099	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3100	if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3101	#if !defined(CONFIG_USER_ONLY)
3102	tb_invalidate_phys_page_fast(ram_addr, 4);
3103	# ifdef VBOX
3104	if (RT_UNLIKELY((ram_addr >> TARGET_PAGE_BITS) >= phys_ram_dirty_size))
3105	dirty_flags = 0xff;
3106	else
3107	# endif /* VBOX */
3108	dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
3109	#endif
3110	}
3111	#if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
3112	remR3PhysWriteU32(ram_addr, val);
3113	#else
3114	stl_p(qemu_get_ram_ptr(ram_addr), val);
3115	#endif
3116	#ifdef CONFIG_KQEMU
3117	if (cpu_single_env->kqemu_enabled &&
3118	(dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
3119	kqemu_modify_page(cpu_single_env, ram_addr);
3120	#endif
3121	dirty_flags \|= (0xff & ~CODE_DIRTY_FLAG);
3122	#ifdef VBOX
3123	if (RT_LIKELY((ram_addr >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3124	#endif
3125	phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
3126	/* we remove the notdirty callback only if the code has been
3127	flushed */
3128	if (dirty_flags == 0xff)
3129	tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3130	}
3131
3132	static CPUReadMemoryFunc *error_mem_read[3] = {
3133	NULL, /* never used */
3134	NULL, /* never used */
3135	NULL, /* never used */
3136	};
3137
3138	static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
3139	notdirty_mem_writeb,
3140	notdirty_mem_writew,
3141	notdirty_mem_writel,
3142	};
3143
3144	/* Generate a debug exception if a watchpoint has been hit. */
3145	static void check_watchpoint(int offset, int len_mask, int flags)
3146	{
3147	CPUState *env = cpu_single_env;
3148	target_ulong pc, cs_base;
3149	TranslationBlock *tb;
3150	target_ulong vaddr;
3151	CPUWatchpoint *wp;
3152	int cpu_flags;
3153
3154	if (env->watchpoint_hit) {
3155	/* We re-entered the check after replacing the TB. Now raise
3156	* the debug interrupt so that is will trigger after the
3157	* current instruction. */
3158	cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3159	return;
3160	}
3161	vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3162	TAILQ_FOREACH(wp, &env->watchpoints, entry) {
3163	if ((vaddr == (wp->vaddr & len_mask) \|\|
3164	(vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3165	wp->flags \|= BP_WATCHPOINT_HIT;
3166	if (!env->watchpoint_hit) {
3167	env->watchpoint_hit = wp;
3168	tb = tb_find_pc(env->mem_io_pc);
3169	if (!tb) {
3170	cpu_abort(env, "check_watchpoint: could not find TB for "
3171	"pc=%p", (void *)env->mem_io_pc);
3172	}
3173	cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3174	tb_phys_invalidate(tb, -1);
3175	if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3176	env->exception_index = EXCP_DEBUG;
3177	} else {
3178	cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3179	tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3180	}
3181	cpu_resume_from_signal(env, NULL);
3182	}
3183	} else {
3184	wp->flags &= ~BP_WATCHPOINT_HIT;
3185	}
3186	}
3187	}
3188
3189	/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3190	so these check for a hit then pass through to the normal out-of-line
3191	phys routines. */
3192	static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3193	{
3194	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3195	return ldub_phys(addr);
3196	}
3197
3198	static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3199	{
3200	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3201	return lduw_phys(addr);
3202	}
3203
3204	static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3205	{
3206	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3207	return ldl_phys(addr);
3208	}
3209
3210	static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3211	uint32_t val)
3212	{
3213	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3214	stb_phys(addr, val);
3215	}
3216
3217	static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3218	uint32_t val)
3219	{
3220	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3221	stw_phys(addr, val);
3222	}
3223
3224	static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3225	uint32_t val)
3226	{
3227	check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3228	stl_phys(addr, val);
3229	}
3230
3231	static CPUReadMemoryFunc *watch_mem_read[3] = {
3232	watch_mem_readb,
3233	watch_mem_readw,
3234	watch_mem_readl,
3235	};
3236
3237	static CPUWriteMemoryFunc *watch_mem_write[3] = {
3238	watch_mem_writeb,
3239	watch_mem_writew,
3240	watch_mem_writel,
3241	};
3242
3243	static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
3244	unsigned int len)
3245	{
3246	uint32_t ret;
3247	unsigned int idx;
3248
3249	idx = SUBPAGE_IDX(addr);
3250	#if defined(DEBUG_SUBPAGE)
3251	printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3252	mmio, len, addr, idx);
3253	#endif
3254	ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
3255	addr + mmio->region_offset[idx][0][len]);
3256
3257	return ret;
3258	}
3259
3260	static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3261	uint32_t value, unsigned int len)
3262	{
3263	unsigned int idx;
3264
3265	idx = SUBPAGE_IDX(addr);
3266	#if defined(DEBUG_SUBPAGE)
3267	printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
3268	mmio, len, addr, idx, value);
3269	#endif
3270	(**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
3271	addr + mmio->region_offset[idx][1][len],
3272	value);
3273	}
3274
3275	static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3276	{
3277	#if defined(DEBUG_SUBPAGE)
3278	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3279	#endif
3280
3281	return subpage_readlen(opaque, addr, 0);
3282	}
3283
3284	static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3285	uint32_t value)
3286	{
3287	#if defined(DEBUG_SUBPAGE)
3288	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3289	#endif
3290	subpage_writelen(opaque, addr, value, 0);
3291	}
3292
3293	static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3294	{
3295	#if defined(DEBUG_SUBPAGE)
3296	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3297	#endif
3298
3299	return subpage_readlen(opaque, addr, 1);
3300	}
3301
3302	static void subpage_writew (void *opaque, target_phys_addr_t addr,
3303	uint32_t value)
3304	{
3305	#if defined(DEBUG_SUBPAGE)
3306	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3307	#endif
3308	subpage_writelen(opaque, addr, value, 1);
3309	}
3310
3311	static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3312	{
3313	#if defined(DEBUG_SUBPAGE)
3314	printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3315	#endif
3316
3317	return subpage_readlen(opaque, addr, 2);
3318	}
3319
3320	static void subpage_writel (void *opaque,
3321	target_phys_addr_t addr, uint32_t value)
3322	{
3323	#if defined(DEBUG_SUBPAGE)
3324	printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3325	#endif
3326	subpage_writelen(opaque, addr, value, 2);
3327	}
3328
3329	static CPUReadMemoryFunc *subpage_read[] = {
3330	&subpage_readb,
3331	&subpage_readw,
3332	&subpage_readl,
3333	};
3334
3335	static CPUWriteMemoryFunc *subpage_write[] = {
3336	&subpage_writeb,
3337	&subpage_writew,
3338	&subpage_writel,
3339	};
3340
3341	static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3342	ram_addr_t memory, ram_addr_t region_offset)
3343	{
3344	int idx, eidx;
3345	unsigned int i;
3346
3347	if (start >= TARGET_PAGE_SIZE \|\| end >= TARGET_PAGE_SIZE)
3348	return -1;
3349	idx = SUBPAGE_IDX(start);
3350	eidx = SUBPAGE_IDX(end);
3351	#if defined(DEBUG_SUBPAGE)
3352	printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3353	mmio, start, end, idx, eidx, memory);
3354	#endif
3355	memory >>= IO_MEM_SHIFT;
3356	for (; idx <= eidx; idx++) {
3357	for (i = 0; i < 4; i++) {
3358	if (io_mem_read[memory][i]) {
3359	mmio->mem_read[idx][i] = &io_mem_read[memory][i];
3360	mmio->opaque[idx][0][i] = io_mem_opaque[memory];
3361	mmio->region_offset[idx][0][i] = region_offset;
3362	}
3363	if (io_mem_write[memory][i]) {
3364	mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3365	mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3366	mmio->region_offset[idx][1][i] = region_offset;
3367	}
3368	}
3369	}
3370
3371	return 0;
3372	}
3373
3374	static void subpage_init (target_phys_addr_t base, ram_addr_t phys,
3375	ram_addr_t orig_memory, ram_addr_t region_offset)
3376	{
3377	subpage_t *mmio;
3378	int subpage_memory;
3379
3380	mmio = qemu_mallocz(sizeof(subpage_t));
3381
3382	mmio->base = base;
3383	subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3384	#if defined(DEBUG_SUBPAGE)
3385	printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3386	mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3387	#endif
3388	*phys = subpage_memory \| IO_MEM_SUBPAGE;
3389	subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3390	region_offset);
3391
3392	return mmio;
3393	}
3394
3395	static int get_free_io_mem_idx(void)
3396	{
3397	int i;
3398
3399	for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3400	if (!io_mem_used[i]) {
3401	io_mem_used[i] = 1;
3402	return i;
3403	}
3404
3405	return -1;
3406	}
3407
3408	/* mem_read and mem_write are arrays of functions containing the
3409	function to access byte (index 0), word (index 1) and dword (index
3410	2). Functions can be omitted with a NULL function pointer.
3411	If io_index is non zero, the corresponding io zone is
3412	modified. If it is zero, a new io zone is allocated. The return
3413	value can be used with cpu_register_physical_memory(). (-1) is
3414	returned if error. */
3415	static int cpu_register_io_memory_fixed(int io_index,
3416	CPUReadMemoryFunc **mem_read,
3417	CPUWriteMemoryFunc **mem_write,
3418	void *opaque)
3419	{
3420	int i, subwidth = 0;
3421
3422	if (io_index <= 0) {
3423	io_index = get_free_io_mem_idx();
3424	if (io_index == -1)
3425	return io_index;
3426	} else {
3427	io_index >>= IO_MEM_SHIFT;
3428	if (io_index >= IO_MEM_NB_ENTRIES)
3429	return -1;
3430	}
3431
3432	for(i = 0;i < 3; i++) {
3433	if (!mem_read[i] \|\| !mem_write[i])
3434	subwidth = IO_MEM_SUBWIDTH;
3435	io_mem_read[io_index][i] = mem_read[i];
3436	io_mem_write[io_index][i] = mem_write[i];
3437	}
3438	io_mem_opaque[io_index] = opaque;
3439	return (io_index << IO_MEM_SHIFT) \| subwidth;
3440	}
3441
3442	int cpu_register_io_memory(CPUReadMemoryFunc **mem_read,
3443	CPUWriteMemoryFunc **mem_write,
3444	void *opaque)
3445	{
3446	return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3447	}
3448
3449	void cpu_unregister_io_memory(int io_table_address)
3450	{
3451	int i;
3452	int io_index = io_table_address >> IO_MEM_SHIFT;
3453
3454	for (i=0;i < 3; i++) {
3455	io_mem_read[io_index][i] = unassigned_mem_read[i];
3456	io_mem_write[io_index][i] = unassigned_mem_write[i];
3457	}
3458	io_mem_opaque[io_index] = NULL;
3459	io_mem_used[io_index] = 0;
3460	}
3461
3462	static void io_mem_init(void)
3463	{
3464	int i;
3465
3466	cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3467	cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3468	cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3469	for (i=0; i<5; i++)
3470	io_mem_used[i] = 1;
3471
3472	io_mem_watch = cpu_register_io_memory(watch_mem_read,
3473	watch_mem_write, NULL);
3474	#ifdef CONFIG_KQEMU
3475	if (kqemu_phys_ram_base) {
3476	/* alloc dirty bits array */
3477	phys_ram_dirty = qemu_vmalloc(kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3478	memset(phys_ram_dirty, 0xff, kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3479	}
3480	#endif
3481	}
3482
3483	#endif /* !defined(CONFIG_USER_ONLY) */
3484
3485	/* physical memory access (slow version, mainly for debug) */
3486	#if defined(CONFIG_USER_ONLY)
3487	void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3488	int len, int is_write)
3489	{
3490	int l, flags;
3491	target_ulong page;
3492	void * p;
3493
3494	while (len > 0) {
3495	page = addr & TARGET_PAGE_MASK;
3496	l = (page + TARGET_PAGE_SIZE) - addr;
3497	if (l > len)
3498	l = len;
3499	flags = page_get_flags(page);
3500	if (!(flags & PAGE_VALID))
3501	return;
3502	if (is_write) {
3503	if (!(flags & PAGE_WRITE))
3504	return;
3505	/* XXX: this code should not depend on lock_user */
3506	if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3507	/* FIXME - should this return an error rather than just fail? */
3508	return;
3509	memcpy(p, buf, l);
3510	unlock_user(p, addr, l);
3511	} else {
3512	if (!(flags & PAGE_READ))
3513	return;
3514	/* XXX: this code should not depend on lock_user */
3515	if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3516	/* FIXME - should this return an error rather than just fail? */
3517	return;
3518	memcpy(buf, p, l);
3519	unlock_user(p, addr, 0);
3520	}
3521	len -= l;
3522	buf += l;
3523	addr += l;
3524	}
3525	}
3526
3527	#else
3528	void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3529	int len, int is_write)
3530	{
3531	int l, io_index;
3532	uint8_t *ptr;
3533	uint32_t val;
3534	target_phys_addr_t page;
3535	unsigned long pd;
3536	PhysPageDesc *p;
3537
3538	while (len > 0) {
3539	page = addr & TARGET_PAGE_MASK;
3540	l = (page + TARGET_PAGE_SIZE) - addr;
3541	if (l > len)
3542	l = len;
3543	p = phys_page_find(page >> TARGET_PAGE_BITS);
3544	if (!p) {
3545	pd = IO_MEM_UNASSIGNED;
3546	} else {
3547	pd = p->phys_offset;
3548	}
3549
3550	if (is_write) {
3551	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3552	target_phys_addr_t addr1 = addr;
3553	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3554	if (p)
3555	addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3556	/* XXX: could force cpu_single_env to NULL to avoid
3557	potential bugs */
3558	if (l >= 4 && ((addr1 & 3) == 0)) {
3559	/* 32 bit write access */
3560	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3561	val = ldl_p(buf);
3562	#else
3563	val = (const uint32_t )buf;
3564	#endif
3565	io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3566	l = 4;
3567	} else if (l >= 2 && ((addr1 & 1) == 0)) {
3568	/* 16 bit write access */
3569	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3570	val = lduw_p(buf);
3571	#else
3572	val = (const uint16_t )buf;
3573	#endif
3574	io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3575	l = 2;
3576	} else {
3577	/* 8 bit write access */
3578	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3579	val = ldub_p(buf);
3580	#else
3581	val = (const uint8_t )buf;
3582	#endif
3583	io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3584	l = 1;
3585	}
3586	} else {
3587	unsigned long addr1;
3588	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3589	/* RAM case */
3590	#ifdef VBOX
3591	remR3PhysWrite(addr1, buf, l); NOREF(ptr);
3592	#else
3593	ptr = qemu_get_ram_ptr(addr1);
3594	memcpy(ptr, buf, l);
3595	#endif
3596	if (!cpu_physical_memory_is_dirty(addr1)) {
3597	/* invalidate code */
3598	tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3599	/* set dirty bit */
3600	#ifdef VBOX
3601	if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
3602	#endif
3603	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3604	(0xff & ~CODE_DIRTY_FLAG);
3605	}
3606	}
3607	} else {
3608	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3609	!(pd & IO_MEM_ROMD)) {
3610	target_phys_addr_t addr1 = addr;
3611	/* I/O case */
3612	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3613	if (p)
3614	addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3615	if (l >= 4 && ((addr1 & 3) == 0)) {
3616	/* 32 bit read access */
3617	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3618	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3619	stl_p(buf, val);
3620	#else
3621	(uint32_t )buf = val;
3622	#endif
3623	l = 4;
3624	} else if (l >= 2 && ((addr1 & 1) == 0)) {
3625	/* 16 bit read access */
3626	val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3627	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3628	stw_p(buf, val);
3629	#else
3630	(uint16_t )buf = val;
3631	#endif
3632	l = 2;
3633	} else {
3634	/* 8 bit read access */
3635	val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3636	#if !defined(VBOX) \|\| !defined(REM_PHYS_ADDR_IN_TLB)
3637	stb_p(buf, val);
3638	#else
3639	(uint8_t )buf = val;
3640	#endif
3641	l = 1;
3642	}
3643	} else {
3644	/* RAM case */
3645	#ifdef VBOX
3646	remR3PhysRead((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), buf, l); NOREF(ptr);
3647	#else
3648	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3649	(addr & ~TARGET_PAGE_MASK);
3650	memcpy(buf, ptr, l);
3651	#endif
3652	}
3653	}
3654	len -= l;
3655	buf += l;
3656	addr += l;
3657	}
3658	}
3659
3660	#ifndef VBOX
3661
3662	/* used for ROM loading : can write in RAM and ROM */
3663	void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3664	const uint8_t *buf, int len)
3665	{
3666	int l;
3667	uint8_t *ptr;
3668	target_phys_addr_t page;
3669	unsigned long pd;
3670	PhysPageDesc *p;
3671
3672	while (len > 0) {
3673	page = addr & TARGET_PAGE_MASK;
3674	l = (page + TARGET_PAGE_SIZE) - addr;
3675	if (l > len)
3676	l = len;
3677	p = phys_page_find(page >> TARGET_PAGE_BITS);
3678	if (!p) {
3679	pd = IO_MEM_UNASSIGNED;
3680	} else {
3681	pd = p->phys_offset;
3682	}
3683
3684	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3685	(pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3686	!(pd & IO_MEM_ROMD)) {
3687	/* do nothing */
3688	} else {
3689	unsigned long addr1;
3690	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3691	/* ROM/RAM case */
3692	ptr = qemu_get_ram_ptr(addr1);
3693	memcpy(ptr, buf, l);
3694	}
3695	len -= l;
3696	buf += l;
3697	addr += l;
3698	}
3699	}
3700
3701	typedef struct {
3702	void *buffer;
3703	target_phys_addr_t addr;
3704	target_phys_addr_t len;
3705	} BounceBuffer;
3706
3707	static BounceBuffer bounce;
3708
3709	typedef struct MapClient {
3710	void *opaque;
3711	void (callback)(void opaque);
3712	LIST_ENTRY(MapClient) link;
3713	} MapClient;
3714
3715	static LIST_HEAD(map_client_list, MapClient) map_client_list
3716	= LIST_HEAD_INITIALIZER(map_client_list);
3717
3718	void cpu_register_map_client(void opaque, void (callback)(void opaque))
3719	{
3720	MapClient client = qemu_malloc(sizeof(client));
3721
3722	client->opaque = opaque;
3723	client->callback = callback;
3724	LIST_INSERT_HEAD(&map_client_list, client, link);
3725	return client;
3726	}
3727
3728	void cpu_unregister_map_client(void *_client)
3729	{
3730	MapClient client = (MapClient )_client;
3731
3732	LIST_REMOVE(client, link);
3733	qemu_free(client);
3734	}
3735
3736	static void cpu_notify_map_clients(void)
3737	{
3738	MapClient *client;
3739
3740	while (!LIST_EMPTY(&map_client_list)) {
3741	client = LIST_FIRST(&map_client_list);
3742	client->callback(client->opaque);
3743	cpu_unregister_map_client(client);
3744	}
3745	}
3746
3747	/* Map a physical memory region into a host virtual address.
3748	* May map a subset of the requested range, given by and returned in *plen.
3749	* May return NULL if resources needed to perform the mapping are exhausted.
3750	* Use only for reads OR writes - not for read-modify-write operations.
3751	* Use cpu_register_map_client() to know when retrying the map operation is
3752	* likely to succeed.
3753	*/
3754	void *cpu_physical_memory_map(target_phys_addr_t addr,
3755	target_phys_addr_t *plen,
3756	int is_write)
3757	{
3758	target_phys_addr_t len = *plen;
3759	target_phys_addr_t done = 0;
3760	int l;
3761	uint8_t *ret = NULL;
3762	uint8_t *ptr;
3763	target_phys_addr_t page;
3764	unsigned long pd;
3765	PhysPageDesc *p;
3766	unsigned long addr1;
3767
3768	while (len > 0) {
3769	page = addr & TARGET_PAGE_MASK;
3770	l = (page + TARGET_PAGE_SIZE) - addr;
3771	if (l > len)
3772	l = len;
3773	p = phys_page_find(page >> TARGET_PAGE_BITS);
3774	if (!p) {
3775	pd = IO_MEM_UNASSIGNED;
3776	} else {
3777	pd = p->phys_offset;
3778	}
3779
3780	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3781	if (done \|\| bounce.buffer) {
3782	break;
3783	}
3784	bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3785	bounce.addr = addr;
3786	bounce.len = l;
3787	if (!is_write) {
3788	cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3789	}
3790	ptr = bounce.buffer;
3791	} else {
3792	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3793	ptr = qemu_get_ram_ptr(addr1);
3794	}
3795	if (!done) {
3796	ret = ptr;
3797	} else if (ret + done != ptr) {
3798	break;
3799	}
3800
3801	len -= l;
3802	addr += l;
3803	done += l;
3804	}
3805	*plen = done;
3806	return ret;
3807	}
3808
3809	/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3810	* Will also mark the memory as dirty if is_write == 1. access_len gives
3811	* the amount of memory that was actually read or written by the caller.
3812	*/
3813	void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3814	int is_write, target_phys_addr_t access_len)
3815	{
3816	if (buffer != bounce.buffer) {
3817	if (is_write) {
3818	ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3819	while (access_len) {
3820	unsigned l;
3821	l = TARGET_PAGE_SIZE;
3822	if (l > access_len)
3823	l = access_len;
3824	if (!cpu_physical_memory_is_dirty(addr1)) {
3825	/* invalidate code */
3826	tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3827	/* set dirty bit */
3828	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3829	(0xff & ~CODE_DIRTY_FLAG);
3830	}
3831	addr1 += l;
3832	access_len -= l;
3833	}
3834	}
3835	return;
3836	}
3837	if (is_write) {
3838	cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3839	}
3840	qemu_free(bounce.buffer);
3841	bounce.buffer = NULL;
3842	cpu_notify_map_clients();
3843	}
3844
3845	#endif /* !VBOX */
3846
3847	/* warning: addr must be aligned */
3848	uint32_t ldl_phys(target_phys_addr_t addr)
3849	{
3850	int io_index;
3851	uint8_t *ptr;
3852	uint32_t val;
3853	unsigned long pd;
3854	PhysPageDesc *p;
3855
3856	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3857	if (!p) {
3858	pd = IO_MEM_UNASSIGNED;
3859	} else {
3860	pd = p->phys_offset;
3861	}
3862
3863	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3864	!(pd & IO_MEM_ROMD)) {
3865	/* I/O case */
3866	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3867	if (p)
3868	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3869	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3870	} else {
3871	/* RAM case */
3872	#ifndef VBOX
3873	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3874	(addr & ~TARGET_PAGE_MASK);
3875	val = ldl_p(ptr);
3876	#else
3877	val = remR3PhysReadU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK)); NOREF(ptr);
3878	#endif
3879	}
3880	return val;
3881	}
3882
3883	/* warning: addr must be aligned */
3884	uint64_t ldq_phys(target_phys_addr_t addr)
3885	{
3886	int io_index;
3887	uint8_t *ptr;
3888	uint64_t val;
3889	unsigned long pd;
3890	PhysPageDesc *p;
3891
3892	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3893	if (!p) {
3894	pd = IO_MEM_UNASSIGNED;
3895	} else {
3896	pd = p->phys_offset;
3897	}
3898
3899	if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3900	!(pd & IO_MEM_ROMD)) {
3901	/* I/O case */
3902	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3903	if (p)
3904	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3905	#ifdef TARGET_WORDS_BIGENDIAN
3906	val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3907	val \|= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3908	#else
3909	val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3910	val \|= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3911	#endif
3912	} else {
3913	/* RAM case */
3914	#ifndef VBOX
3915	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3916	(addr & ~TARGET_PAGE_MASK);
3917	val = ldq_p(ptr);
3918	#else
3919	val = remR3PhysReadU64((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK)); NOREF(ptr);
3920	#endif
3921	}
3922	return val;
3923	}
3924
3925	/* XXX: optimize */
3926	uint32_t ldub_phys(target_phys_addr_t addr)
3927	{
3928	uint8_t val;
3929	cpu_physical_memory_read(addr, &val, 1);
3930	return val;
3931	}
3932
3933	/* XXX: optimize */
3934	uint32_t lduw_phys(target_phys_addr_t addr)
3935	{
3936	uint16_t val;
3937	cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3938	return tswap16(val);
3939	}
3940
3941	/* warning: addr must be aligned. The ram page is not masked as dirty
3942	and the code inside is not invalidated. It is useful if the dirty
3943	bits are used to track modified PTEs */
3944	void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3945	{
3946	int io_index;
3947	uint8_t *ptr;
3948	unsigned long pd;
3949	PhysPageDesc *p;
3950
3951	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3952	if (!p) {
3953	pd = IO_MEM_UNASSIGNED;
3954	} else {
3955	pd = p->phys_offset;
3956	}
3957
3958	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3959	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3960	if (p)
3961	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3962	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3963	} else {
3964	#ifndef VBOX
3965	unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3966	ptr = qemu_get_ram_ptr(addr1);
3967	stl_p(ptr, val);
3968	#else
3969	remR3PhysWriteU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
3970	#endif
3971
3972	#ifndef VBOX
3973	if (unlikely(in_migration)) {
3974	if (!cpu_physical_memory_is_dirty(addr1)) {
3975	/* invalidate code */
3976	tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3977	/* set dirty bit */
3978	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
3979	(0xff & ~CODE_DIRTY_FLAG);
3980	}
3981	}
3982	#endif /* !VBOX */
3983	}
3984	}
3985
3986	void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3987	{
3988	int io_index;
3989	uint8_t *ptr;
3990	unsigned long pd;
3991	PhysPageDesc *p;
3992
3993	p = phys_page_find(addr >> TARGET_PAGE_BITS);
3994	if (!p) {
3995	pd = IO_MEM_UNASSIGNED;
3996	} else {
3997	pd = p->phys_offset;
3998	}
3999
4000	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4001	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4002	if (p)
4003	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4004	#ifdef TARGET_WORDS_BIGENDIAN
4005	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4006	io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4007	#else
4008	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4009	io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4010	#endif
4011	} else {
4012	#ifndef VBOX
4013	ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4014	(addr & ~TARGET_PAGE_MASK);
4015	stq_p(ptr, val);
4016	#else
4017	remR3PhysWriteU64((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
4018	#endif
4019	}
4020	}
4021
4022	/* warning: addr must be aligned */
4023	void stl_phys(target_phys_addr_t addr, uint32_t val)
4024	{
4025	int io_index;
4026	uint8_t *ptr;
4027	unsigned long pd;
4028	PhysPageDesc *p;
4029
4030	p = phys_page_find(addr >> TARGET_PAGE_BITS);
4031	if (!p) {
4032	pd = IO_MEM_UNASSIGNED;
4033	} else {
4034	pd = p->phys_offset;
4035	}
4036
4037	if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4038	io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4039	if (p)
4040	addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4041	io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4042	} else {
4043	unsigned long addr1;
4044	addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4045	/* RAM case */
4046	#ifndef VBOX
4047	ptr = qemu_get_ram_ptr(addr1);
4048	stl_p(ptr, val);
4049	#else
4050	remR3PhysWriteU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
4051	#endif
4052	if (!cpu_physical_memory_is_dirty(addr1)) {
4053	/* invalidate code */
4054	tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4055	/* set dirty bit */
4056	#ifdef VBOX
4057	if (RT_LIKELY((addr1 >> TARGET_PAGE_BITS) < phys_ram_dirty_size))
4058	#endif
4059	phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] \|=
4060	(0xff & ~CODE_DIRTY_FLAG);
4061	}
4062	}
4063	}
4064
4065	/* XXX: optimize */
4066	void stb_phys(target_phys_addr_t addr, uint32_t val)
4067	{
4068	uint8_t v = val;
4069	cpu_physical_memory_write(addr, &v, 1);
4070	}
4071
4072	/* XXX: optimize */
4073	void stw_phys(target_phys_addr_t addr, uint32_t val)
4074	{
4075	uint16_t v = tswap16(val);
4076	cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
4077	}
4078
4079	/* XXX: optimize */
4080	void stq_phys(target_phys_addr_t addr, uint64_t val)
4081	{
4082	val = tswap64(val);
4083	cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
4084	}
4085
4086	#endif
4087
4088	#ifndef VBOX
4089	/* virtual memory access for debug (includes writing to ROM) */
4090	int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4091	uint8_t *buf, int len, int is_write)
4092	{
4093	int l;
4094	target_phys_addr_t phys_addr;
4095	target_ulong page;
4096
4097	while (len > 0) {
4098	page = addr & TARGET_PAGE_MASK;
4099	phys_addr = cpu_get_phys_page_debug(env, page);
4100	/* if no physical page mapped, return an error */
4101	if (phys_addr == -1)
4102	return -1;
4103	l = (page + TARGET_PAGE_SIZE) - addr;
4104	if (l > len)
4105	l = len;
4106	phys_addr += (addr & ~TARGET_PAGE_MASK);
4107	#if !defined(CONFIG_USER_ONLY)
4108	if (is_write)
4109	cpu_physical_memory_write_rom(phys_addr, buf, l);
4110	else
4111	#endif
4112	cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4113	len -= l;
4114	buf += l;
4115	addr += l;
4116	}
4117	return 0;
4118	}
4119	#endif /* !VBOX */
4120
4121	/* in deterministic execution mode, instructions doing device I/Os
4122	must be at the end of the TB */
4123	void cpu_io_recompile(CPUState env, void retaddr)
4124	{
4125	TranslationBlock *tb;
4126	uint32_t n, cflags;
4127	target_ulong pc, cs_base;
4128	uint64_t flags;
4129
4130	tb = tb_find_pc((unsigned long)retaddr);
4131	if (!tb) {
4132	cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4133	retaddr);
4134	}
4135	n = env->icount_decr.u16.low + tb->icount;
4136	cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
4137	/* Calculate how many instructions had been executed before the fault
4138	occurred. */
4139	n = n - env->icount_decr.u16.low;
4140	/* Generate a new TB ending on the I/O insn. */
4141	n++;
4142	/* On MIPS and SH, delay slot instructions can only be restarted if
4143	they were already the first instruction in the TB. If this is not
4144	the first instruction in a TB then re-execute the preceding
4145	branch. */
4146	#if defined(TARGET_MIPS)
4147	if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4148	env->active_tc.PC -= 4;
4149	env->icount_decr.u16.low++;
4150	env->hflags &= ~MIPS_HFLAG_BMASK;
4151	}
4152	#elif defined(TARGET_SH4)
4153	if ((env->flags & ((DELAY_SLOT \| DELAY_SLOT_CONDITIONAL))) != 0
4154	&& n > 1) {
4155	env->pc -= 2;
4156	env->icount_decr.u16.low++;
4157	env->flags &= ~(DELAY_SLOT \| DELAY_SLOT_CONDITIONAL);
4158	}
4159	#endif
4160	/* This should never happen. */
4161	if (n > CF_COUNT_MASK)
4162	cpu_abort(env, "TB too big during recompile");
4163
4164	cflags = n \| CF_LAST_IO;
4165	pc = tb->pc;
4166	cs_base = tb->cs_base;
4167	flags = tb->flags;
4168	tb_phys_invalidate(tb, -1);
4169	/* FIXME: In theory this could raise an exception. In practice
4170	we have already translated the block once so it's probably ok. */
4171	tb_gen_code(env, pc, cs_base, flags, cflags);
4172	/* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4173	the first in the TB) then we end up generating a whole new TB and
4174	repeating the fault, which is horribly inefficient.
4175	Better would be to execute just this insn uncached, or generate a
4176	second new TB. */
4177	cpu_resume_from_signal(env, NULL);
4178	}
4179
4180	#ifndef VBOX
4181	void dump_exec_info(FILE *f,
4182	int (cpu_fprintf)(FILE f, const char *fmt, ...))
4183	{
4184	int i, target_code_size, max_target_code_size;
4185	int direct_jmp_count, direct_jmp2_count, cross_page;
4186	TranslationBlock *tb;
4187
4188	target_code_size = 0;
4189	max_target_code_size = 0;
4190	cross_page = 0;
4191	direct_jmp_count = 0;
4192	direct_jmp2_count = 0;
4193	for(i = 0; i < nb_tbs; i++) {
4194	tb = &tbs[i];
4195	target_code_size += tb->size;
4196	if (tb->size > max_target_code_size)
4197	max_target_code_size = tb->size;
4198	if (tb->page_addr[1] != -1)
4199	cross_page++;
4200	if (tb->tb_next_offset[0] != 0xffff) {
4201	direct_jmp_count++;
4202	if (tb->tb_next_offset[1] != 0xffff) {
4203	direct_jmp2_count++;
4204	}
4205	}
4206	}
4207	/* XXX: avoid using doubles ? */
4208	cpu_fprintf(f, "Translation buffer state:\n");
4209	cpu_fprintf(f, "gen code size %ld/%ld\n",
4210	code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4211	cpu_fprintf(f, "TB count %d/%d\n",
4212	nb_tbs, code_gen_max_blocks);
4213	cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4214	nb_tbs ? target_code_size / nb_tbs : 0,
4215	max_target_code_size);
4216	cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4217	nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4218	target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4219	cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4220	cross_page,
4221	nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4222	cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4223	direct_jmp_count,
4224	nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4225	direct_jmp2_count,
4226	nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4227	cpu_fprintf(f, "\nStatistics:\n");
4228	cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4229	cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4230	cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4231	tcg_dump_info(f, cpu_fprintf);
4232	}
4233	#endif /* !VBOX */
4234
4235	#if !defined(CONFIG_USER_ONLY)
4236
4237	#define MMUSUFFIX _cmmu
4238	#define GETPC() NULL
4239	#define env cpu_single_env
4240	#define SOFTMMU_CODE_ACCESS
4241
4242	#define SHIFT 0
4243	#include "softmmu_template.h"
4244
4245	#define SHIFT 1
4246	#include "softmmu_template.h"
4247
4248	#define SHIFT 2
4249	#include "softmmu_template.h"
4250
4251	#define SHIFT 3
4252	#include "softmmu_template.h"
4253
4254	#undef env
4255
4256	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/recompiler/exec.c@ 36211

Download in other formats: