IEMAllThreadedPython.py@ 98951

Last change on this file since 98951 was 98951, checked in by vboxsync, 2 years ago
VMM/IEM: More work on processing MC blocks and generating functions from them. bugref:10369
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 41.1 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: IEMAllThreadedPython.py 98951 2023-03-14 10:34:58Z vboxsync $
4	# pylint: disable=invalid-name
5
6	"""
7	Annotates and generates threaded functions from IEMAllInstructions*.cpp.h.
8	"""
9
10	from __future__ import print_function;
11
12	__copyright__ = \
13	"""
14	Copyright (C) 2023 Oracle and/or its affiliates.
15
16	This file is part of VirtualBox base platform packages, as
17	available from https://www.215389.xyz.
18
19	This program is free software; you can redistribute it and/or
20	modify it under the terms of the GNU General Public License
21	as published by the Free Software Foundation, in version 3 of the
22	License.
23
24	This program is distributed in the hope that it will be useful, but
25	WITHOUT ANY WARRANTY; without even the implied warranty of
26	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
27	General Public License for more details.
28
29	You should have received a copy of the GNU General Public License
30	along with this program; if not, see <https://www.gnu.org/licenses>.
31
32	SPDX-License-Identifier: GPL-3.0-only
33	"""
34	__version__ = "$Revision: 98951 $"
35
36	# Standard python imports.
37	import copy;
38	import datetime;
39	import os;
40	import sys;
41	import argparse;
42
43	import IEMAllInstructionsPython as iai;
44
45
46	# Python 3 hacks:
47	if sys.version_info[0] >= 3:
48	long = int; # pylint: disable=redefined-builtin,invalid-name
49
50	## Number of generic parameters for the thread functions.
51	g_kcThreadedParams = 3;
52
53	g_kdTypeInfo = {
54	# type name: (cBits, fSigned, C-type )
55	'int8_t': ( 8, True, 'uint8_t', ),
56	'int16_t': ( 16, True, 'int16_t', ),
57	'int32_t': ( 32, True, 'int32_t', ),
58	'int64_t': ( 64, True, 'int64_t', ),
59	'uint4_t': ( 4, False, 'uint8_t', ),
60	'uint8_t': ( 8, False, 'uint8_t', ),
61	'uint16_t': ( 16, False, 'uint16_t', ),
62	'uint32_t': ( 32, False, 'uint32_t', ),
63	'uint64_t': ( 64, False, 'uint64_t', ),
64	'uintptr_t': ( 64, False, 'uintptr_t', ), # ASSUMES 64-bit host pointer size.
65	'bool': ( 1, False, 'bool', ),
66	'IEMMODE': ( 2, False, 'IEMMODE', ),
67	};
68
69	g_kdIemFieldToType = {
70	# Illegal ones:
71	'offInstrNextByte': ( None, ),
72	'cbInstrBuf': ( None, ),
73	'pbInstrBuf': ( None, ),
74	'uInstrBufPc': ( None, ),
75	'cbInstrBufTotal': ( None, ),
76	'offCurInstrStart': ( None, ),
77	'cbOpcode': ( None, ),
78	'offOpcode': ( None, ),
79	'offModRm': ( None, ),
80	# Okay ones.
81	'fPrefixes': ( 'uint32_t', ),
82	'uRexReg': ( 'uint8_t', ),
83	'uRexB': ( 'uint8_t', ),
84	'uRexIndex': ( 'uint8_t', ),
85	'iEffSeg': ( 'uint8_t', ),
86	'enmEffOpSize': ( 'IEMMODE', ),
87	'enmDefAddrMode': ( 'IEMMODE', ),
88	'enmEffAddrMode': ( 'IEMMODE', ),
89	'enmDefOpSize': ( 'IEMMODE', ),
90	'idxPrefix': ( 'uint8_t', ),
91	'uVex3rdReg': ( 'uint8_t', ),
92	'uVexLength': ( 'uint8_t', ),
93	'fEvexStuff': ( 'uint8_t', ),
94	'uFpuOpcode': ( 'uint16_t', ),
95	};
96
97	class ThreadedParamRef(object):
98	"""
99	A parameter reference for a threaded function.
100	"""
101
102	def __init__(self, sOrgRef, sType, oStmt, iParam = None, offParam = 0):
103	self.sOrgRef = sOrgRef; ##< The name / reference in the original code.
104	self.sStdRef = ''.join(sOrgRef.split()); ##< Normalized name to deal with spaces in macro invocations and such.
105	self.sType = sType; ##< The type (typically derived).
106	self.oStmt = oStmt; ##< The statement making the reference.
107	self.iParam = iParam; ##< The parameter containing the references. None if implicit.
108	self.offParam = offParam; ##< The offset in the parameter of the reference.
109
110	self.sNewName = 'x'; ##< The variable name in the threaded function.
111	self.iNewParam = 99; ##< The this is packed into.
112	self.offNewParam = 1024 ##< The bit offset in iNewParam.
113
114	class ThreadedFunction(object):
115	"""
116	A threaded function.
117	"""
118
119	def __init__(self, oMcBlock):
120	self.oMcBlock = oMcBlock # type: IEMAllInstructionsPython.McBlock
121	## Dictionary of local variables (IEM_MC_LOCAL[_CONST]) and call arguments (IEM_MC_ARG*).
122	self.dVariables = {} # type: dict(str,McStmtVar)
123	##
124	self.aoParamRefs = [] # type: list(ThreadedParamRef)
125	self.dParamRefs = {} # type: dict(str,list(ThreadedParamRef))
126	self.cMinParams = 0; ##< Minimum number of parameters to the threaded function.
127
128	## List/tree of statements for the threaded function.
129	self.aoStmtsForThreadedFunction = [] # type list(McStmt)
130
131	@staticmethod
132	def dummyInstance():
133	""" Gets a dummy instance. """
134	return ThreadedFunction(iai.McBlock('null', 999999999, 999999999, 'nil', 999999999));
135
136	def raiseProblem(self, sMessage):
137	""" Raises a problem. """
138	raise Exception('%s:%s: error: %s' % (self.oMcBlock.sSrcFile, self.oMcBlock.iBeginLine, sMessage, ));
139
140	def getIndexName(self):
141	sName = self.oMcBlock.sFunction;
142	if sName.startswith('iemOp_'):
143	sName = sName[len('iemOp_'):];
144	if self.oMcBlock.iInFunction == 0:
145	return 'kIemThreadedFunc_%s' % ( sName, );
146	return 'kIemThreadedFunc_%s_%s' % ( sName, self.oMcBlock.iInFunction, );
147
148	def getFunctionName(self):
149	sName = self.oMcBlock.sFunction;
150	if sName.startswith('iemOp_'):
151	sName = sName[len('iemOp_'):];
152	if self.oMcBlock.iInFunction == 0:
153	return 'iemThreadedFunc_%s' % ( sName, );
154	return 'iemThreadedFunc_%s_%s' % ( sName, self.oMcBlock.iInFunction, );
155
156	def analyzeReferenceToType(self, sRef):
157	"""
158	Translates a variable or structure reference to a type.
159	Returns type name.
160	Raises exception if unable to figure it out.
161	"""
162	ch0 = sRef[0];
163	if ch0 == 'u':
164	if sRef.startswith('u32'):
165	return 'uint32_t';
166	if sRef.startswith('u8') or sRef == 'uReg':
167	return 'uint8_t';
168	if sRef.startswith('u64'):
169	return 'uint64_t';
170	if sRef.startswith('u16'):
171	return 'uint16_t';
172	elif ch0 == 'b':
173	return 'uint8_t';
174	elif ch0 == 'f':
175	return 'bool';
176	elif ch0 == 'i':
177	if sRef.startswith('i8'):
178	return 'int8_t';
179	if sRef.startswith('i16'):
180	return 'int32_t';
181	if sRef.startswith('i32'):
182	return 'int32_t';
183	if sRef.startswith('i64'):
184	return 'int64_t';
185	if sRef in ('iReg', 'iSegReg', 'iSrcReg', 'iDstReg'):
186	return 'uint8_t';
187	elif ch0 == 'p':
188	if sRef.find('-') < 0:
189	return 'uintptr_t';
190	if sRef.startswith('pVCpu->iem.s.'):
191	sField = sRef[len('pVCpu->iem.s.') : ];
192	if sField in g_kdIemFieldToType:
193	if g_kdIemFieldToType[sField][0]:
194	return g_kdIemFieldToType[sField][0];
195	self.raiseProblem('Reference out-of-bounds decoder field: %s' % (sRef,));
196	elif ch0 == 'G' and sRef.startswith('GCPtr'):
197	return 'uint64_t';
198	elif sRef == 'cShift': ## @todo risky
199	return 'uint8_t';
200	self.raiseProblem('Unknown reference: %s' % (sRef,));
201	return None; # Shut up pylint 2.16.2.
202
203	def analyzeMorphStmtForThreaded(self, aoStmts, iParamRef = 0):
204	"""
205	Transforms (copy) the statements into those for the threaded function.
206
207	Returns list/tree of statements (aoStmts is not modified) and the new
208	iParamRef value.
209	"""
210	#
211	# We'll be traversing aoParamRefs in parallel to the statements, so we
212	# must match the traversal in analyzeFindThreadedParamRefs exactly.
213	#
214	#print('McBlock at %s:%s' % (os.path.split(self.oMcBlock.sSrcFile)[1], self.oMcBlock.iBeginLine,));
215	aoThreadedStmts = [];
216	for oStmt in aoStmts:
217	# Skip C++ statements that is purely related to decoding.
218	if not oStmt.isCppStmt() or not oStmt.fDecode:
219	# Copy the statement. Make a deep copy to make sure we've got our own
220	# copies of all instance variables, even if a bit overkill at the moment.
221	oNewStmt = copy.deepcopy(oStmt);
222	aoThreadedStmts.append(oNewStmt);
223	#print('oNewStmt %s %s' % (oNewStmt.sName, len(oNewStmt.asParams),));
224
225	# If the statement has parameter references, process the relevant parameters.
226	# We grab the references relevant to this statement and apply them in reserve order.
227	if iParamRef < len(self.aoParamRefs) and self.aoParamRefs[iParamRef].oStmt == oStmt:
228	iParamRefFirst = iParamRef;
229	while True:
230	iParamRef += 1;
231	if iParamRef >= len(self.aoParamRefs) or self.aoParamRefs[iParamRef].oStmt != oStmt:
232	break;
233
234	#print('iParamRefFirst=%s iParamRef=%s' % (iParamRefFirst, iParamRef));
235	for iCurRef in range(iParamRef - 1, iParamRefFirst - 1, -1):
236	oCurRef = self.aoParamRefs[iCurRef];
237	if oCurRef.iParam is not None:
238	assert oCurRef.oStmt == oStmt;
239	#print('iCurRef=%s iParam=%s sOrgRef=%s' % (iCurRef, oCurRef.iParam, oCurRef.sOrgRef));
240	sSrcParam = oNewStmt.asParams[oCurRef.iParam];
241	assert sSrcParam[oCurRef.offParam : oCurRef.offParam + len(oCurRef.sOrgRef)] == oCurRef.sOrgRef, \
242	'offParam=%s sOrgRef=%s sSrcParam=%s<eos>' % (oCurRef.offParam, oCurRef.sOrgRef, sSrcParam);
243	oNewStmt.asParams[oCurRef.iParam] = sSrcParam[0 : oCurRef.offParam] \
244	+ oCurRef.sNewName \
245	+ sSrcParam[oCurRef.offParam + len(oCurRef.sOrgRef) : ];
246
247	# Process branches of conditionals recursively.
248	if isinstance(oStmt, iai.McStmtCond):
249	(oNewStmt.aoIfBranch, iParamRef) = self.analyzeMorphStmtForThreaded(oStmt.aoIfBranch, iParamRef);
250	if oStmt.aoElseBranch:
251	(oNewStmt.aoElseBranch, iParamRef) = self.analyzeMorphStmtForThreaded(oStmt.aoElseBranch, iParamRef);
252
253	return (aoThreadedStmts, iParamRef);
254
255	def analyzeConsolidateThreadedParamRefs(self):
256	"""
257	Consolidate threaded function parameter references into a dictionary
258	with lists of the references to each variable/field.
259	"""
260	# Gather unique parameters.
261	self.dParamRefs = {};
262	for oRef in self.aoParamRefs:
263	if oRef.sStdRef not in self.dParamRefs:
264	self.dParamRefs[oRef.sStdRef] = [oRef,];
265	else:
266	self.dParamRefs[oRef.sStdRef].append(oRef);
267
268	# Generate names for them for use in the threaded function.
269	dParamNames = {};
270	for sName, aoRefs in self.dParamRefs.items():
271	# Morph the reference expression into a name.
272	if sName.startswith('IEM_GET_MODRM_REG'): sName = 'bModRmRegP';
273	elif sName.startswith('IEM_GET_MODRM_RM'): sName = 'bModRmRmP';
274	elif sName.startswith('IEM_GET_MODRM_REG_8'): sName = 'bModRmReg8P';
275	elif sName.startswith('IEM_GET_MODRM_RM_8'): sName = 'bModRmRm8P';
276	elif sName.startswith('IEM_GET_EFFECTIVE_VVVV'): sName = 'bEffVvvvP';
277	elif sName.find('.') >= 0 or sName.find('->') >= 0:
278	sName = sName[max(sName.rfind('.'), sName.rfind('>')) + 1 : ] + 'P';
279	else:
280	sName += 'P';
281
282	# Ensure it's unique.
283	if sName in dParamNames:
284	for i in range(10):
285	if sName + str(i) not in dParamNames:
286	sName += str(i);
287	break;
288	dParamNames[sName] = True;
289
290	# Update all the references.
291	for oRef in aoRefs:
292	oRef.sNewName = sName;
293
294	# Organize them by size too for the purpose of optimize them.
295	dBySize = {} # type: dict(str,str)
296	for sStdRef, aoRefs in self.dParamRefs.items():
297	cBits = g_kdTypeInfo[aoRefs[0].sType][0];
298	assert(cBits <= 64);
299	if cBits not in dBySize:
300	dBySize[cBits] = [sStdRef,]
301	else:
302	dBySize[cBits].append(sStdRef);
303
304	# Pack the parameters as best as we can, starting with the largest ones
305	# and ASSUMING a 64-bit parameter size.
306	self.cMinParams = 0;
307	offNewParam = 0;
308	for cBits in sorted(dBySize.keys(), reverse = True):
309	for sStdRef in dBySize[cBits]:
310	if offNewParam < 64:
311	offNewParam += cBits;
312	else:
313	self.cMinParams += 1;
314	offNewParam = cBits;
315	assert(offNewParam <= 64);
316
317	for oRef in self.dParamRefs[sStdRef]:
318	oRef.iNewParam = self.cMinParams;
319	oRef.offNewParam = offNewParam - cBits;
320
321	if offNewParam > 0:
322	self.cMinParams += 1;
323
324	# Currently there are a few that requires 4 parameters, list these so we can figure out why:
325	if self.cMinParams >= 3:
326	print('debug: cMinParams=%s cRawParams=%s - %s:%d'
327	% (self.cMinParams, len(self.dParamRefs), self.oMcBlock.sSrcFile, self.oMcBlock.iBeginLine,));
328
329	return True;
330
331	ksHexDigits = '0123456789abcdefABCDEF';
332
333	def analyzeFindThreadedParamRefs(self, aoStmts):
334	"""
335	Scans the statements for things that have to passed on to the threaded
336	function (populates self.aoParamRefs).
337	"""
338	for oStmt in aoStmts:
339	# Some statements we can skip alltogether.
340	if isinstance(oStmt, (iai.McStmtVar, iai.McCppPreProc)):
341	continue;
342	if oStmt.isCppStmt() and oStmt.fDecode:
343	continue;
344
345	# Several statements have implicit parameters.
346	if oStmt.sName in ('IEM_MC_ADVANCE_RIP_AND_FINISH', 'IEM_MC_REL_JMP_S8_AND_FINISH', 'IEM_MC_REL_JMP_S16_AND_FINISH',
347	'IEM_MC_REL_JMP_S32_AND_FINISH', 'IEM_MC_CALL_CIMPL_0', 'IEM_MC_CALL_CIMPL_1',
348	'IEM_MC_CALL_CIMPL_2', 'IEM_MC_CALL_CIMPL_3', 'IEM_MC_CALL_CIMPL_4', 'IEM_MC_CALL_CIMPL_5'):
349	self.aoParamRefs.append(ThreadedParamRef('cbInstr', 'uint4_t', oStmt));
350
351	# We can skip the rest for statements w/o parameters.
352	if not oStmt.asParams:
353	continue;
354
355	# Inspect the target of calls to see if we need to pass down a
356	# function pointer or function table pointer for it to work.
357	aiSkipParams = {};
358	if isinstance(oStmt, iai.McStmtCall):
359	if oStmt.sFn[0] == 'p':
360	self.aoParamRefs.append(ThreadedParamRef(oStmt.sFn, 'uintptr_t', oStmt, oStmt.idxFn));
361	elif ( oStmt.sFn[0] != 'i'
362	and not oStmt.sFn.startswith('IEMTARGETCPU_EFL_BEHAVIOR_SELECT')
363	and not oStmt.sFn.startswith('IEM_SELECT_HOST_OR_FALLBACK') ):
364	self.raiseProblem('Bogus function name in %s: %s' % (oStmt.sName, oStmt.sFn,));
365	aiSkipParams[oStmt.idxFn] = True;
366
367	# Check all the parameters for bogus references.
368	for iParam, sParam in enumerate(oStmt.asParams):
369	if iParam not in aiSkipParams and sParam not in self.dVariables:
370	# The parameter may contain a C expression, so we have to try
371	# extract the relevant bits, i.e. variables and fields while
372	# ignoring operators and parentheses.
373	offParam = 0;
374	while offParam < len(sParam):
375	# Is it the start of an C identifier? If so, find the end, but don't stop on field separators (->, .).
376	ch = sParam[offParam];
377	if ch.isalpha() or ch == '_':
378	offStart = offParam;
379	offParam += 1;
380	while offParam < len(sParam):
381	ch = sParam[offParam];
382	if not ch.isalnum() and ch != '_' and ch != '.':
383	if ch != '-' or sParam[offParam + 1] != '>':
384	# Special hack for the 'CTX_SUFF(pVM)' bit in pVCpu->CTX_SUFF(pVM)->xxxx:
385	if ( ch == '('
386	and sParam[offStart : offParam + len('(pVM)->')] == 'pVCpu->CTX_SUFF(pVM)->'):
387	offParam += len('(pVM)->') - 1;
388	else:
389	break;
390	offParam += 1;
391	offParam += 1;
392	sRef = sParam[offStart : offParam];
393
394	# For register references, we pass the full register indexes instead as macros
395	# like IEM_GET_MODRM_REG implicitly references pVCpu->iem.s.uRexReg and the
396	# threaded function will be more efficient if we just pass the register index
397	# as a 4-bit param.
398	if ( sRef.startswith('IEM_GET_MODRM')
399	or sRef.startswith('IEM_GET_EFFECTIVE_VVVV') ):
400	offParam = iai.McBlock.skipSpacesAt(sParam, offParam, len(sParam));
401	if sParam[offParam] != '(':
402	self.raiseProblem('Expected "(" following %s in "%s"' % (sRef, oStmt.renderCode(),));
403	(asMacroParams, offCloseParam) = iai.McBlock.extractParams(sParam, offParam);
404	if asMacroParams is None:
405	self.raiseProblem('Unable to find ")" for %s in "%s"' % (sRef, oStmt.renderCode(),));
406	offParam = offCloseParam + 1;
407	self.aoParamRefs.append(ThreadedParamRef(sParam[offStart : offParam], 'uint8_t',
408	oStmt, iParam, offStart));
409
410	# We can skip known variables.
411	elif sRef in self.dVariables:
412	pass;
413
414	# Skip certain macro invocations.
415	elif sRef in ('IEM_GET_HOST_CPU_FEATURES',
416	'IEM_GET_GUEST_CPU_FEATURES',
417	'IEM_IS_GUEST_CPU_AMD'):
418	offParam = iai.McBlock.skipSpacesAt(sParam, offParam, len(sParam));
419	if sParam[offParam] != '(':
420	self.raiseProblem('Expected "(" following %s in "%s"' % (sRef, oStmt.renderCode(),));
421	(asMacroParams, offCloseParam) = iai.McBlock.extractParams(sParam, offParam);
422	if asMacroParams is None:
423	self.raiseProblem('Unable to find ")" for %s in "%s"' % (sRef, oStmt.renderCode(),));
424	offParam = offCloseParam + 1;
425	while offParam < len(sParam) and (sParam[offParam].isalnum() or sParam[offParam] in '_.'):
426	offParam += 1;
427
428	# Skip constants, globals, types (casts), sizeof and macros.
429	elif ( sRef.startswith('IEM_OP_PRF_')
430	or sRef.startswith('IEM_ACCESS_')
431	or sRef.startswith('X86_GREG_')
432	or sRef.startswith('X86_SREG_')
433	or sRef.startswith('X86_EFL_')
434	or sRef.startswith('X86_FSW_')
435	or sRef.startswith('X86_FCW_')
436	or sRef.startswith('g_')
437	or sRef in ( 'int8_t', 'int16_t', 'int32_t',
438	'INT8_C', 'INT16_C', 'INT32_C', 'INT64_C',
439	'UINT8_C', 'UINT16_C', 'UINT32_C', 'UINT64_C',
440	'UINT8_MAX', 'UINT16_MAX', 'UINT32_MAX', 'UINT64_MAX',
441	'sizeof', 'NOREF', 'RT_NOREF', 'IEMMODE_64BIT' ) ):
442	pass;
443
444	# Skip certain macro invocations.
445	# Any variable (non-field) and decoder fields in IEMCPU will need to be parameterized.
446	elif ( ( '.' not in sRef
447	and '-' not in sRef
448	and sRef not in ('pVCpu', ) )
449	or iai.McBlock.koReIemDecoderVars.search(sRef) is not None):
450	self.aoParamRefs.append(ThreadedParamRef(sRef, self.analyzeReferenceToType(sRef),
451	oStmt, iParam, offStart));
452	# Number.
453	elif ch.isdigit():
454	if ( ch == '0'
455	and offParam + 2 <= len(sParam)
456	and sParam[offParam + 1] in 'xX'
457	and sParam[offParam + 2] in self.ksHexDigits ):
458	offParam += 2;
459	while offParam < len(sParam) and sParam[offParam] in self.ksHexDigits:
460	offParam += 1;
461	else:
462	while offParam < len(sParam) and sParam[offParam].isdigit():
463	offParam += 1;
464	# Whatever else.
465	else:
466	offParam += 1;
467
468	# Traverse the branches of conditionals.
469	if isinstance(oStmt, iai.McStmtCond):
470	self.analyzeFindThreadedParamRefs(oStmt.aoIfBranch);
471	self.analyzeFindThreadedParamRefs(oStmt.aoElseBranch);
472	return True;
473
474	def analyzeFindVariablesAndCallArgs(self, aoStmts):
475	""" Scans the statements for MC variables and call arguments. """
476	for oStmt in aoStmts:
477	if isinstance(oStmt, iai.McStmtVar):
478	if oStmt.sVarName in self.dVariables:
479	raise Exception('Variable %s is defined more than once!' % (oStmt.sVarName,));
480	self.dVariables[oStmt.sVarName] = oStmt.sVarName;
481
482	# There shouldn't be any variables or arguments declared inside if/
483	# else blocks, but scan them too to be on the safe side.
484	if isinstance(oStmt, iai.McStmtCond):
485	cBefore = len(self.dVariables);
486	self.analyzeFindVariablesAndCallArgs(oStmt.aoIfBranch);
487	self.analyzeFindVariablesAndCallArgs(oStmt.aoElseBranch);
488	if len(self.dVariables) != cBefore:
489	raise Exception('Variables/arguments defined in conditional branches!');
490	return True;
491
492	def analyze(self):
493	"""
494	Analyzes the code, identifying the number of parameters it requires and such.
495	May raise exceptions if we cannot grok the code.
496	"""
497
498	# Decode the block into a list/tree of McStmt objects.
499	aoStmts = self.oMcBlock.decode();
500
501	# Scan the statements for local variables and call arguments (self.dVariables).
502	self.analyzeFindVariablesAndCallArgs(aoStmts);
503
504	# Now scan the code for variables and field references that needs to
505	# be passed to the threaded function because they are related to the
506	# instruction decoding.
507	self.analyzeFindThreadedParamRefs(aoStmts);
508	self.analyzeConsolidateThreadedParamRefs();
509
510	# Morph the statement stream for the block into what we'll be using in the threaded function.
511	(self.aoStmtsForThreadedFunction, iParamRef) = self.analyzeMorphStmtForThreaded(aoStmts);
512	if iParamRef != len(self.aoParamRefs):
513	raise Exception('iParamRef=%s, expected %s!' % (iParamRef, len(self.aoParamRefs),));
514
515	return True;
516
517	def generateInputCode(self):
518	"""
519	Modifies the input code.
520	"""
521	assert len(self.oMcBlock.asLines) > 2, "asLines=%s" % (self.oMcBlock.asLines,);
522	cchIndent = (self.oMcBlock.cchIndent + 3) // 4 * 4;
523	return iai.McStmt.renderCodeForList(self.oMcBlock.aoStmts, cchIndent = cchIndent).replace('\n', ' /* gen */\n', 1);
524
525
526	class IEMThreadedGenerator(object):
527	"""
528	The threaded code generator & annotator.
529	"""
530
531	def __init__(self):
532	self.aoThreadedFuncs = [] # type: list(ThreadedFunction)
533	self.oOptions = None # type: argparse.Namespace
534	self.aoParsers = [] # type: list(IEMAllInstructionsPython.SimpleParser)
535
536	#
537	# Processing.
538	#
539
540	def processInputFiles(self):
541	"""
542	Process the input files.
543	"""
544
545	# Parse the files.
546	self.aoParsers = iai.parseFiles(self.oOptions.asInFiles);
547
548	# Wrap MC blocks into threaded functions and analyze these.
549	self.aoThreadedFuncs = [ThreadedFunction(oMcBlock) for oMcBlock in iai.g_aoMcBlocks];
550	dRawParamCounts = {};
551	dMinParamCounts = {};
552	for oThreadedFunction in self.aoThreadedFuncs:
553	oThreadedFunction.analyze();
554	dRawParamCounts[len(oThreadedFunction.dParamRefs)] = dRawParamCounts.get(len(oThreadedFunction.dParamRefs), 0) + 1;
555	dMinParamCounts[oThreadedFunction.cMinParams] = dMinParamCounts.get(oThreadedFunction.cMinParams, 0) + 1;
556	print('debug: param count distribution, raw and optimized:', file = sys.stderr);
557	for cCount in sorted({cBits: True for cBits in list(dRawParamCounts.keys()) + list(dMinParamCounts.keys())}.keys()):
558	print('debug: %s params: %4s raw, %4s min'
559	% (cCount, dRawParamCounts.get(cCount, 0), dMinParamCounts.get(cCount, 0)),
560	file = sys.stderr);
561
562	return True;
563
564	#
565	# Output
566	#
567
568	def generateLicenseHeader(self):
569	"""
570	Returns the lines for a license header.
571	"""
572	return [
573	'/*',
574	' * Autogenerated by $Id: IEMAllThreadedPython.py 98951 2023-03-14 10:34:58Z vboxsync $ ',
575	' * Do not edit!',
576	' */',
577	'',
578	'/*',
579	' * Copyright (C) 2023-' + str(datetime.date.today().year) + ' Oracle and/or its affiliates.',
580	' *',
581	' * This file is part of VirtualBox base platform packages, as',
582	' * available from https://www.215389.xyz.',
583	' *',
584	' * This program is free software; you can redistribute it and/or',
585	' * modify it under the terms of the GNU General Public License',
586	' * as published by the Free Software Foundation, in version 3 of the',
587	' * License.',
588	' *',
589	' * This program is distributed in the hope that it will be useful, but',
590	' * WITHOUT ANY WARRANTY; without even the implied warranty of',
591	' * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU',
592	' * General Public License for more details.',
593	' *',
594	' * You should have received a copy of the GNU General Public License',
595	' * along with this program; if not, see <https://www.gnu.org/licenses>.',
596	' *',
597	' * The contents of this file may alternatively be used under the terms',
598	' * of the Common Development and Distribution License Version 1.0',
599	' * (CDDL), a copy of it is provided in the "COPYING.CDDL" file included',
600	' * in the VirtualBox distribution, in which case the provisions of the',
601	' * CDDL are applicable instead of those of the GPL.',
602	' *',
603	' * You may elect to license modified versions of this file under the',
604	' * terms and conditions of either the GPL or the CDDL or both.',
605	' *',
606	' * SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0',
607	' */',
608	'',
609	'',
610	'',
611	];
612
613
614	def generateThreadedFunctionsHeader(self, oOut):
615	"""
616	Generates the threaded functions header file.
617	Returns success indicator.
618	"""
619
620	asLines = self.generateLicenseHeader();
621
622	# Generate the threaded function table indexes.
623	asLines += [
624	'typedef enum IEMTHREADEDFUNCS',
625	'{',
626	' kIemThreadedFunc_Invalid = 0,',
627	];
628	for oFunc in self.aoThreadedFuncs:
629	asLines.append(' ' + oFunc.getIndexName() + ',');
630	asLines += [
631	' kIemThreadedFunc_End',
632	'} IEMTHREADEDFUNCS;',
633	'',
634	];
635
636	# Prototype the function table.
637	sFnType = 'typedef IEM_DECL_IMPL_TYPE(VBOXSTRICTRC, FNIEMTHREADEDFUNC, (PVMCPU pVCpu';
638	for iParam in range(g_kcThreadedParams):
639	sFnType += ', uint64_t uParam' + str(iParam);
640	sFnType += '));'
641
642	asLines += [
643	sFnType,
644	'typedef FNIEMTHREADEDFUNC *PFNIEMTHREADEDFUNC;',
645	'',
646	'extern const PFNIEMTHREADEDFUNC g_apfnIemThreadedFunctions[kIemThreadedFunc_End];',
647	];
648
649	oOut.write('\n'.join(asLines));
650	return True;
651
652	ksBitsToIntMask = {
653	1: "UINT64_C(0x1)",
654	2: "UINT64_C(0x3)",
655	4: "UINT64_C(0xf)",
656	8: "UINT64_C(0xff)",
657	16: "UINT64_C(0xffff)",
658	32: "UINT64_C(0xffffffff)",
659	};
660	def generateThreadedFunctionsSource(self, oOut):
661	"""
662	Generates the threaded functions source file.
663	Returns success indicator.
664	"""
665
666	asLines = self.generateLicenseHeader();
667	oOut.write('\n'.join(asLines));
668
669	# Prepare the fixed bits.
670	sParamList = '(PVMCPU pVCpu';
671	for iParam in range(g_kcThreadedParams):
672	sParamList += ', uint64_t uParam' + str(iParam);
673	sParamList += '))\n';
674
675	#
676	# Emit the function definitions.
677	#
678	for oThreadedFunction in self.aoThreadedFuncs:
679	oMcBlock = oThreadedFunction.oMcBlock;
680	# Function header
681	oOut.write( '\n'
682	+ '\n'
683	+ '/**\n'
684	+ ' * %s at line %s offset %s in %s%s\n'
685	% (oMcBlock.sFunction, oMcBlock.iBeginLine, oMcBlock.offBeginLine, os.path.split(oMcBlock.sSrcFile)[1],
686	' (macro expansion)' if oMcBlock.iBeginLine == oMcBlock.iEndLine else '')
687	+ ' */\n'
688	+ 'static IEM_DECL_IMPL_DEF(VBOXSTRICTRC, ' + oThreadedFunction.getFunctionName() + ',\n'
689	+ ' ' + sParamList
690	+ '{\n');
691
692	aasVars = [];
693	for aoRefs in oThreadedFunction.dParamRefs.values():
694	oRef = aoRefs[0];
695	cBits = g_kdTypeInfo[oRef.sType][0];
696
697	sTypeDecl = oRef.sType + ' const';
698
699	if cBits == 64:
700	assert oRef.offNewParam == 0;
701	if oRef.sType == 'uint64_t':
702	sUnpack = 'uParam%s;' % (oRef.iNewParam,);
703	else:
704	sUnpack = '(%s)uParam%s;' % (oRef.sType, oRef.iNewParam,);
705	elif oRef.offNewParam == 0:
706	sUnpack = '(%s)(uParam%s & %s);' % (oRef.sType, oRef.iNewParam, self.ksBitsToIntMask[cBits]);
707	else:
708	sUnpack = '(%s)((uParam%s >> %s) & %s);' \
709	% (oRef.sType, oRef.iNewParam, oRef.offNewParam, self.ksBitsToIntMask[cBits]);
710
711	sComment = '/* %s - %s ref%s */' % (oRef.sOrgRef, len(aoRefs), 's' if len(aoRefs) != 1 else '',);
712
713	aasVars.append([ '%s:%02u' % (oRef.iNewParam, oRef.offNewParam), sTypeDecl, oRef.sNewName, sUnpack, sComment ]);
714	acchVars = [0, 0, 0, 0, 0];
715	for asVar in aasVars:
716	for iCol, sStr in enumerate(asVar):
717	acchVars[iCol] = max(acchVars[iCol], len(sStr));
718	sFmt = ' %%-%ss %%-%ss = %%-%ss %%s\n' % (acchVars[1], acchVars[2], acchVars[3]);
719	for asVar in sorted(aasVars):
720	oOut.write(sFmt % (asVar[1], asVar[2], asVar[3], asVar[4],));
721
722	# RT_NOREF for unused parameters.
723	if oThreadedFunction.cMinParams < g_kcThreadedParams:
724	oOut.write(' RT_NOREF('
725	+ ', '.join(['uParam%u' % (i,) for i in range(oThreadedFunction.cMinParams, g_kcThreadedParams)])
726	+ ');\n');
727
728	# Now for the actual statements.
729	oOut.write(iai.McStmt.renderCodeForList(oThreadedFunction.aoStmtsForThreadedFunction, cchIndent = 4));
730
731	oOut.write('}\n');
732
733	#
734	# Emit the function table.
735	#
736	oOut.write( '\n'
737	+ '\n'
738	+ '/**\n'
739	+ ' * Function table.\n'
740	+ ' */\n'
741	+ 'const PFNIEMTHREADEDFUNC g_apfnIemThreadedFunctions[kIemThreadedFunc_End] =\n'
742	+ '{\n'
743	+ ' /Invalid/ NULL, \n');
744	for iThreadedFunction, oThreadedFunction in enumerate(self.aoThreadedFuncs):
745	oOut.write(' /%4u/ %s,\n' % (iThreadedFunction + 1, oThreadedFunction.getFunctionName(),));
746	oOut.write('};\n');
747
748	return True;
749
750	def getThreadedFunctionByIndex(self, idx):
751	"""
752	Returns a ThreadedFunction object for the given index. If the index is
753	out of bounds, a dummy is returned.
754	"""
755	if idx < len(self.aoThreadedFuncs):
756	return self.aoThreadedFuncs[idx];
757	return ThreadedFunction.dummyInstance();
758
759	def findEndOfMcEndStmt(self, sLine, offEndStmt):
760	"""
761	Helper that returns the line offset following the 'IEM_MC_END();'.
762	"""
763	assert sLine[offEndStmt:].startswith('IEM_MC_END');
764	off = sLine.find(';', offEndStmt + len('IEM_MC_END'));
765	assert off > 0, 'sLine="%s"' % (sLine, );
766	return off + 1 if off > 0 else 99999998;
767
768	def generateModifiedInput(self, oOut):
769	"""
770	Generates the combined modified input source/header file.
771	Returns success indicator.
772	"""
773	#
774	# File header.
775	#
776	oOut.write('\n'.join(self.generateLicenseHeader()));
777
778	#
779	# ASSUMING that g_aoMcBlocks/self.aoThreadedFuncs are in self.aoParsers
780	# order, we iterate aoThreadedFuncs in parallel to the lines from the
781	# parsers and apply modifications where required.
782	#
783	iThreadedFunction = 0;
784	oThreadedFunction = self.getThreadedFunctionByIndex(0);
785	for oParser in self.aoParsers: # IEMAllInstructionsPython.SimpleParser
786	oOut.write("\n\n/* **** BEGIN %s ***** */\n" % (oParser.sSrcFile,));
787
788	iLine = 0;
789	while iLine < len(oParser.asLines):
790	sLine = oParser.asLines[iLine];
791	iLine += 1; # iBeginLine and iEndLine are 1-based.
792
793	# Can we pass it thru?
794	if ( iLine not in [oThreadedFunction.oMcBlock.iBeginLine, oThreadedFunction.oMcBlock.iEndLine]
795	or oThreadedFunction.oMcBlock.sSrcFile != oParser.sSrcFile):
796	oOut.write(sLine);
797	#
798	# Single MC block. Just extract it and insert the replacement.
799	#
800	elif oThreadedFunction.oMcBlock.iBeginLine != oThreadedFunction.oMcBlock.iEndLine:
801	assert sLine.count('IEM_MC_') == 1;
802	oOut.write(sLine[:oThreadedFunction.oMcBlock.offBeginLine]);
803	sModified = oThreadedFunction.generateInputCode().strip();
804	oOut.write(sModified);
805
806	iLine = oThreadedFunction.oMcBlock.iEndLine;
807	sLine = oParser.asLines[iLine - 1];
808	assert sLine.count('IEM_MC_') == 1;
809	oOut.write(sLine[self.findEndOfMcEndStmt(sLine, oThreadedFunction.oMcBlock.offEndLine) : ]);
810
811	# Advance
812	iThreadedFunction += 1;
813	oThreadedFunction = self.getThreadedFunctionByIndex(iThreadedFunction);
814	#
815	# Macro expansion line that have sublines and may contain multiple MC blocks.
816	#
817	else:
818	offLine = 0;
819	while iLine == oThreadedFunction.oMcBlock.iBeginLine:
820	oOut.write(sLine[offLine : oThreadedFunction.oMcBlock.offBeginLine]);
821
822	sModified = oThreadedFunction.generateInputCode().strip();
823	assert sModified.startswith('IEM_MC_BEGIN'), 'sModified="%s"' % (sModified,);
824	oOut.write(sModified);
825
826	offLine = self.findEndOfMcEndStmt(sLine, oThreadedFunction.oMcBlock.offEndLine);
827
828	# Advance
829	iThreadedFunction += 1;
830	oThreadedFunction = self.getThreadedFunctionByIndex(iThreadedFunction);
831
832	# Last line segment.
833	if offLine < len(sLine):
834	oOut.write(sLine[offLine : ]);
835
836	oOut.write("/* **** END %s ***** */\n" % (oParser.sSrcFile,));
837
838	return True;
839
840	#
841	# Main
842	#
843
844	def main(self, asArgs):
845	"""
846	C-like main function.
847	Returns exit code.
848	"""
849
850	#
851	# Parse arguments
852	#
853	sScriptDir = os.path.dirname(__file__);
854	oParser = argparse.ArgumentParser(add_help = False);
855	oParser.add_argument('asInFiles', metavar = 'input.cpp.h', nargs = '*',
856	default = [os.path.join(sScriptDir, asFM[0]) for asFM in iai.g_aasAllInstrFilesAndDefaultMap],
857	help = "Selection of VMMAll/IEMAllInstructions*.cpp.h files to use as input.");
858	oParser.add_argument('--out-funcs-hdr', metavar = 'file-funcs.h', dest = 'sOutFileFuncsHdr', action = 'store',
859	default = '-', help = 'The output header file for the functions.');
860	oParser.add_argument('--out-funcs-cpp', metavar = 'file-funcs.cpp', dest = 'sOutFileFuncsCpp', action = 'store',
861	default = '-', help = 'The output C++ file for the functions.');
862	oParser.add_argument('--out-mod-input', metavar = 'file-instr.cpp.h', dest = 'sOutFileModInput', action = 'store',
863	default = '-', help = 'The output C++/header file for the modified input instruction files.');
864	oParser.add_argument('--help', '-h', '-?', action = 'help', help = 'Display help and exit.');
865	oParser.add_argument('--version', '-V', action = 'version',
866	version = 'r%s (IEMAllThreadedPython.py), r%s (IEMAllInstructionsPython.py)'
867	% (__version__.split()[1], iai.__version__.split()[1],),
868	help = 'Displays the version/revision of the script and exit.');
869	self.oOptions = oParser.parse_args(asArgs[1:]);
870	print("oOptions=%s" % (self.oOptions,));
871
872	#
873	# Process the instructions specified in the IEM sources.
874	#
875	if self.processInputFiles():
876	#
877	# Generate the output files.
878	#
879	aaoOutputFiles = (
880	( self.oOptions.sOutFileFuncsHdr, self.generateThreadedFunctionsHeader ),
881	( self.oOptions.sOutFileFuncsCpp, self.generateThreadedFunctionsSource ),
882	( self.oOptions.sOutFileModInput, self.generateModifiedInput ),
883	);
884	fRc = True;
885	for sOutFile, fnGenMethod in aaoOutputFiles:
886	if sOutFile == '-':
887	fRc = fnGenMethod(sys.stdout) and fRc;
888	else:
889	try:
890	oOut = open(sOutFile, 'w'); # pylint: disable=consider-using-with,unspecified-encoding
891	except Exception as oXcpt:
892	print('error! Failed open "%s" for writing: %s' % (sOutFile, oXcpt,), file = sys.stderr);
893	return 1;
894	fRc = fnGenMethod(oOut) and fRc;
895	oOut.close();
896	if fRc:
897	return 0;
898
899	return 1;
900
901
902	if __name__ == '__main__':
903	sys.exit(IEMThreadedGenerator().main(sys.argv));
904

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllThreadedPython.py@ 98951

Download in other formats: