utils.py@ 65006

Last change on this file since 65006 was 65006, checked in by vboxsync, 8 years ago
common/utils.py: Use 'ps' on solaris so we get info on suid processes and other users' processes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 58.2 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 65006 2016-12-23 16:41:30Z vboxsync $
3	# pylint: disable=C0302
4
5	"""
6	Common Utility Functions.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2016 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.215389.xyz. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 65006 $"
31
32
33	# Standard Python imports.
34	import datetime;
35	import os;
36	import platform;
37	import re;
38	import stat;
39	import subprocess;
40	import sys;
41	import time;
42	import traceback;
43	import unittest;
44
45	if sys.platform == 'win32':
46	import ctypes;
47	import win32api; # pylint: disable=F0401
48	import win32con; # pylint: disable=F0401
49	import win32console; # pylint: disable=F0401
50	import win32process; # pylint: disable=F0401
51	else:
52	import signal;
53
54	# Python 3 hacks:
55	if sys.version_info[0] >= 3:
56	unicode = str; # pylint: disable=redefined-builtin,invalid-name
57	xrange = range; # pylint: disable=redefined-builtin,invalid-name
58	long = int; # pylint: disable=redefined-builtin,invalid-name
59
60
61	#
62	# Host OS and CPU.
63	#
64
65	def getHostOs():
66	"""
67	Gets the host OS name (short).
68
69	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
70	"""
71	sPlatform = platform.system();
72	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
73	sPlatform = sPlatform.lower();
74	elif sPlatform == 'Windows':
75	sPlatform = 'win';
76	elif sPlatform == 'SunOS':
77	sPlatform = 'solaris';
78	else:
79	raise Exception('Unsupported platform "%s"' % (sPlatform,));
80	return sPlatform;
81
82	g_sHostArch = None;
83
84	def getHostArch():
85	"""
86	Gets the host CPU architecture.
87
88	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
89	"""
90	global g_sHostArch;
91	if g_sHostArch is None:
92	sArch = platform.machine();
93	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
94	sArch = 'x86';
95	elif sArch in ('AMD64', 'amd64', 'x86_64'):
96	sArch = 'amd64';
97	elif sArch == 'i86pc': # SunOS
98	if platform.architecture()[0] == '64bit':
99	sArch = 'amd64';
100	else:
101	try:
102	sArch = processOutputChecked(['/usr/bin/isainfo', '-n',]);
103	except:
104	pass;
105	sArch = sArch.strip();
106	if sArch != 'amd64':
107	sArch = 'x86';
108	else:
109	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
110	g_sHostArch = sArch;
111	return g_sHostArch;
112
113
114	def getHostOsDotArch():
115	"""
116	Gets the 'os.arch' for the host.
117	"""
118	return '%s.%s' % (getHostOs(), getHostArch());
119
120
121	def isValidOs(sOs):
122	"""
123	Validates the OS name.
124	"""
125	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
126	'os2', 'solaris', 'win', 'os-agnostic'):
127	return True;
128	return False;
129
130
131	def isValidArch(sArch):
132	"""
133	Validates the CPU architecture name.
134	"""
135	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
136	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
137	return True;
138	return False;
139
140	def isValidOsDotArch(sOsDotArch):
141	"""
142	Validates the 'os.arch' string.
143	"""
144
145	asParts = sOsDotArch.split('.');
146	if asParts.length() != 2:
147	return False;
148	return isValidOs(asParts[0]) \
149	and isValidArch(asParts[1]);
150
151	def getHostOsVersion():
152	"""
153	Returns the host OS version. This is platform.release with additional
154	distro indicator on linux.
155	"""
156	sVersion = platform.release();
157	sOs = getHostOs();
158	if sOs == 'linux':
159	sDist = '';
160	try:
161	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
162	oFile = open('/etc/lsb-release');
163	for sLine in oFile:
164	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
165	if oMatch is not None:
166	sDist = oMatch.group(1).strip();
167	except:
168	pass;
169	if sDist:
170	sVersion += ' / ' + sDist;
171	else:
172	asFiles = \
173	[
174	[ '/etc/debian_version', 'Debian v'],
175	[ '/etc/gentoo-release', '' ],
176	[ '/etc/oracle-release', '' ],
177	[ '/etc/redhat-release', '' ],
178	[ '/etc/SuSE-release', '' ],
179	];
180	for sFile, sPrefix in asFiles:
181	if os.path.isfile(sFile):
182	try:
183	oFile = open(sFile);
184	sLine = oFile.readline();
185	oFile.close();
186	except:
187	continue;
188	sLine = sLine.strip()
189	if len(sLine) > 0:
190	sVersion += ' / ' + sPrefix + sLine;
191	break;
192
193	elif sOs == 'solaris':
194	sVersion = platform.version();
195	if os.path.isfile('/etc/release'):
196	try:
197	oFile = open('/etc/release');
198	sLast = oFile.readlines()[-1];
199	oFile.close();
200	sLast = sLast.strip();
201	if len(sLast) > 0:
202	sVersion += ' (' + sLast + ')';
203	except:
204	pass;
205
206	elif sOs == 'darwin':
207	sOsxVersion = platform.mac_ver()[0];
208	codenames = {"4": "Tiger",
209	"5": "Leopard",
210	"6": "Snow Leopard",
211	"7": "Lion",
212	"8": "Mountain Lion",
213	"9": "Mavericks",
214	"10": "Yosemite",
215	"11": "El Capitan",
216	"12": "Sierra",
217	"13": "Unknown 13",
218	"14": "Unknown 14", }
219	sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
220
221	return sVersion;
222
223	#
224	# File system.
225	#
226
227	def openNoInherit(sFile, sMode = 'r'):
228	"""
229	Wrapper around open() that tries it's best to make sure the file isn't
230	inherited by child processes.
231
232	This is a best effort thing at the moment as it doesn't synchronizes with
233	child process spawning in any way. Thus it can be subject to races in
234	multithreaded programs.
235	"""
236
237	try:
238	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=F0401
239	except:
240	return open(sFile, sMode);
241
242	oFile = open(sFile, sMode)
243	#try:
244	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
245	#except:
246	# pass;
247	return oFile;
248
249	def noxcptReadLink(sPath, sXcptRet):
250	"""
251	No exceptions os.readlink wrapper.
252	"""
253	try:
254	sRet = os.readlink(sPath); # pylint: disable=E1101
255	except:
256	sRet = sXcptRet;
257	return sRet;
258
259	def readFile(sFile, sMode = 'rb'):
260	"""
261	Reads the entire file.
262	"""
263	oFile = open(sFile, sMode);
264	sRet = oFile.read();
265	oFile.close();
266	return sRet;
267
268	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb'):
269	"""
270	No exceptions common.readFile wrapper.
271	"""
272	try:
273	sRet = readFile(sFile, sMode);
274	except:
275	sRet = sXcptRet;
276	return sRet;
277
278	def noxcptRmDir(sDir, oXcptRet = False):
279	"""
280	No exceptions os.rmdir wrapper.
281	"""
282	oRet = True;
283	try:
284	os.rmdir(sDir);
285	except:
286	oRet = oXcptRet;
287	return oRet;
288
289	def noxcptDeleteFile(sFile, oXcptRet = False):
290	"""
291	No exceptions os.remove wrapper.
292	"""
293	oRet = True;
294	try:
295	os.remove(sFile);
296	except:
297	oRet = oXcptRet;
298	return oRet;
299
300
301	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
302	# type: (string, (string, stat) -> bool) -> bool
303	"""
304	Recursively walks a directory tree, calling fnCallback for each.
305
306	fnCallback takes a full path and stat object (can be None). It
307	returns a boolean value, False stops walking and returns immediately.
308
309	Returns True or False depending on fnCallback.
310	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
311	"""
312	def __worker(sCurDir):
313	""" Worker for """
314	try:
315	asNames = os.listdir(sCurDir);
316	except:
317	if not fIgnoreExceptions:
318	raise;
319	return None;
320	rc = True;
321	for sName in asNames:
322	if sName not in [ '.', '..' ]:
323	sFullName = os.path.join(sCurDir, sName);
324	try: oStat = os.lstat(sFullName);
325	except: oStat = None;
326	if fnCallback(sFullName, oStat) is False:
327	return False;
328	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
329	rc = __worker(sFullName);
330	if rc is False:
331	break;
332	return rc;
333
334	# Ensure unicode path here so listdir also returns unicode on windows.
335	## @todo figure out unicode stuff on non-windows.
336	if sys.platform == 'win32':
337	sDir = unicode(sDir);
338	return __worker(sDir);
339
340
341
342	def formatFileMode(uMode):
343	# type: (int) -> string
344	"""
345	Format a st_mode value 'ls -la' fasion.
346	Returns string.
347	"""
348	if stat.S_ISDIR(uMode): sMode = 'd';
349	elif stat.S_ISREG(uMode): sMode = '-';
350	elif stat.S_ISLNK(uMode): sMode = 'l';
351	elif stat.S_ISFIFO(uMode): sMode = 'p';
352	elif stat.S_ISCHR(uMode): sMode = 'c';
353	elif stat.S_ISBLK(uMode): sMode = 'b';
354	elif stat.S_ISSOCK(uMode): sMode = 's';
355	else: sMode = '?';
356	## @todo sticky bits.
357	sMode += 'r' if uMode & stat.S_IRUSR else '-';
358	sMode += 'w' if uMode & stat.S_IWUSR else '-';
359	sMode += 'x' if uMode & stat.S_IXUSR else '-';
360	sMode += 'r' if uMode & stat.S_IRGRP else '-';
361	sMode += 'w' if uMode & stat.S_IWGRP else '-';
362	sMode += 'x' if uMode & stat.S_IXGRP else '-';
363	sMode += 'r' if uMode & stat.S_IROTH else '-';
364	sMode += 'w' if uMode & stat.S_IWOTH else '-';
365	sMode += 'x' if uMode & stat.S_IXOTH else '-';
366	sMode += ' ';
367	return sMode;
368
369
370	def formatFileStat(oStat):
371	# type: (stat) -> string
372	"""
373	Format a stat result 'ls -la' fasion (numeric IDs).
374	Returns string.
375	"""
376	return '%s %3s %4s %4s %10s %s' \
377	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
378	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
379
380	## Good buffer for file operations.
381	g_cbGoodBufferSize = 256*1024;
382
383	## The original shutil.copyfileobj.
384	g_fnOriginalShCopyFileObj = None;
385
386	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
387	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
388	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
389
390	def __installShUtilHacks(shutil):
391	""" Installs the shutil buffer size hacks. """
392	global g_fnOriginalShCopyFileObj;
393	if g_fnOriginalShCopyFileObj is None:
394	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
395	shutil.copyfileobj = __myshutilcopyfileobj;
396	return True;
397
398
399	def copyFileSimple(sFileSrc, sFileDst):
400	"""
401	Wrapper around shutil.copyfile that simply copies the data of a regular file.
402	Raises exception on failure.
403	Return True for show.
404	"""
405	import shutil;
406	__installShUtilHacks(shutil);
407	return shutil.copyfile(sFileSrc, sFileDst);
408
409	#
410	# SubProcess.
411	#
412
413	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
414	"""
415	If the "executable" is a python script, insert the python interpreter at
416	the head of the argument list so that it will work on systems which doesn't
417	support hash-bang scripts.
418	"""
419
420	asArgs = dKeywordArgs.get('args');
421	if asArgs is None:
422	asArgs = aPositionalArgs[0];
423
424	if asArgs[0].endswith('.py'):
425	if sys.executable is not None and len(sys.executable) > 0:
426	asArgs.insert(0, sys.executable);
427	else:
428	asArgs.insert(0, 'python');
429
430	# paranoia...
431	if dKeywordArgs.get('args') is not None:
432	dKeywordArgs['args'] = asArgs;
433	else:
434	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
435	return None;
436
437	def processCall(aPositionalArgs, *dKeywordArgs):
438	"""
439	Wrapper around subprocess.call to deal with its absense in older
440	python versions.
441	Returns process exit code (see subprocess.poll).
442	"""
443	assert dKeywordArgs.get('stdout') is None;
444	assert dKeywordArgs.get('stderr') is None;
445	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
446	oProcess = subprocess.Popen(aPositionalArgs, *dKeywordArgs);
447	return oProcess.wait();
448
449	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
450	"""
451	Wrapper around subprocess.check_output to deal with its absense in older
452	python versions.
453	"""
454	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
455	oProcess = subprocess.Popen(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
456
457	sOutput, _ = oProcess.communicate();
458	iExitCode = oProcess.poll();
459
460	if iExitCode is not 0:
461	asArgs = dKeywordArgs.get('args');
462	if asArgs is None:
463	asArgs = aPositionalArgs[0];
464	print(sOutput);
465	raise subprocess.CalledProcessError(iExitCode, asArgs);
466
467	return str(sOutput); # str() make pylint happy.
468
469	g_fOldSudo = None;
470	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
471	"""
472	Adds 'sudo' (or similar) to the args parameter, whereever it is.
473	"""
474
475	# Are we root?
476	fIsRoot = True;
477	try:
478	fIsRoot = os.getuid() == 0; # pylint: disable=E1101
479	except:
480	pass;
481
482	# If not, prepend sudo (non-interactive, simulate initial login).
483	if fIsRoot is not True:
484	asArgs = dKeywordArgs.get('args');
485	if asArgs is None:
486	asArgs = aPositionalArgs[0];
487
488	# Detect old sudo.
489	global g_fOldSudo;
490	if g_fOldSudo is None:
491	try:
492	sVersion = processOutputChecked(['sudo', '-V']);
493	except:
494	sVersion = '1.7.0';
495	sVersion = sVersion.strip().split('\n')[0];
496	sVersion = sVersion.replace('Sudo version', '').strip();
497	g_fOldSudo = len(sVersion) >= 4 \
498	and sVersion[0] == '1' \
499	and sVersion[1] == '.' \
500	and sVersion[2] <= '6' \
501	and sVersion[3] == '.';
502
503	asArgs.insert(0, 'sudo');
504	if not g_fOldSudo:
505	asArgs.insert(1, '-n');
506	if fInitialEnv and not g_fOldSudo:
507	asArgs.insert(1, '-i');
508
509	# paranoia...
510	if dKeywordArgs.get('args') is not None:
511	dKeywordArgs['args'] = asArgs;
512	else:
513	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
514	return None;
515
516
517	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
518	"""
519	sudo (or similar) + subprocess.call
520	"""
521	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
522	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
523	return processCall(aPositionalArgs, *dKeywordArgs);
524
525	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
526	"""
527	sudo (or similar) + subprocess.check_output.
528	"""
529	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
530	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
531	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
532
533	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
534	"""
535	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
536	"""
537	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
538	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
539	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
540
541	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
542	"""
543	sudo (or similar) + subprocess.Popen.
544	"""
545	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
546	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
547	return subprocess.Popen(aPositionalArgs, *dKeywordArgs);
548
549
550	#
551	# Generic process stuff.
552	#
553
554	def processInterrupt(uPid):
555	"""
556	Sends a SIGINT or equivalent to interrupt the specified process.
557	Returns True on success, False on failure.
558
559	On Windows hosts this may not work unless the process happens to be a
560	process group leader.
561	"""
562	if sys.platform == 'win32':
563	try:
564	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, uPid); # pylint: disable=no-member
565	fRc = True;
566	except:
567	fRc = False;
568	else:
569	try:
570	os.kill(uPid, signal.SIGINT);
571	fRc = True;
572	except:
573	fRc = False;
574	return fRc;
575
576	def sendUserSignal1(uPid):
577	"""
578	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
579	(VBoxSVC) or something.
580	Returns True on success, False on failure or if not supported (win).
581
582	On Windows hosts this may not work unless the process happens to be a
583	process group leader.
584	"""
585	if sys.platform == 'win32':
586	fRc = False;
587	else:
588	try:
589	os.kill(uPid, signal.SIGUSR1); # pylint: disable=E1101
590	fRc = True;
591	except:
592	fRc = False;
593	return fRc;
594
595	def processTerminate(uPid):
596	"""
597	Terminates the process in a nice manner (SIGTERM or equivalent).
598	Returns True on success, False on failure.
599	"""
600	fRc = False;
601	if sys.platform == 'win32':
602	try:
603	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, False, uPid); # pylint: disable=no-member
604	except:
605	pass;
606	else:
607	try:
608	win32process.TerminateProcess(hProcess, 0x40010004); # DBG_TERMINATE_PROCESS # pylint: disable=no-member
609	fRc = True;
610	except:
611	pass;
612	win32api.CloseHandle(hProcess) # pylint: disable=no-member
613	else:
614	try:
615	os.kill(uPid, signal.SIGTERM);
616	fRc = True;
617	except:
618	pass;
619	return fRc;
620
621	def processKill(uPid):
622	"""
623	Terminates the process with extreme prejudice (SIGKILL).
624	Returns True on success, False on failure.
625	"""
626	if sys.platform == 'win32':
627	fRc = processTerminate(uPid);
628	else:
629	try:
630	os.kill(uPid, signal.SIGKILL); # pylint: disable=E1101
631	fRc = True;
632	except:
633	fRc = False;
634	return fRc;
635
636	def processKillWithNameCheck(uPid, sName):
637	"""
638	Like processKill(), but checks if the process name matches before killing
639	it. This is intended for killing using potentially stale pid values.
640
641	Returns True on success, False on failure.
642	"""
643
644	if processCheckPidAndName(uPid, sName) is not True:
645	return False;
646	return processKill(uPid);
647
648
649	def processExists(uPid):
650	"""
651	Checks if the specified process exits.
652	This will only work if we can signal/open the process.
653
654	Returns True if it positively exists, False otherwise.
655	"""
656	if sys.platform == 'win32':
657	fRc = False;
658	try:
659	hProcess = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, False, uPid); # pylint: disable=no-member
660	except:
661	pass;
662	else:
663	win32api.CloseHandle(hProcess); # pylint: disable=no-member
664	fRc = True;
665	else:
666	try:
667	os.kill(uPid, 0);
668	fRc = True;
669	except:
670	fRc = False;
671	return fRc;
672
673	def processCheckPidAndName(uPid, sName):
674	"""
675	Checks if a process PID and NAME matches.
676	"""
677	fRc = processExists(uPid);
678	if fRc is not True:
679	return False;
680
681	if sys.platform == 'win32':
682	try:
683	from win32com.client import GetObject; # pylint: disable=F0401
684	oWmi = GetObject('winmgmts:');
685	aoProcesses = oWmi.InstancesOf('Win32_Process');
686	for oProcess in aoProcesses:
687	if long(oProcess.Properties_("ProcessId").Value) == uPid:
688	sCurName = oProcess.Properties_("Name").Value;
689	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
690	sName = sName.lower();
691	sCurName = sCurName.lower();
692	if os.path.basename(sName) == sName:
693	sCurName = os.path.basename(sCurName);
694
695	if sCurName == sName \
696	or sCurName + '.exe' == sName \
697	or sCurName == sName + '.exe':
698	fRc = True;
699	break;
700	except:
701	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
702	pass;
703	else:
704	if sys.platform in ('linux2', ):
705	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
706	elif sys.platform in ('sunos5',):
707	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
708	elif sys.platform in ('darwin',):
709	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
710	else:
711	asPsCmd = None;
712
713	if asPsCmd is not None:
714	try:
715	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
716	sCurName = oPs.communicate()[0];
717	iExitCode = oPs.wait();
718	except:
719	#reporter.logXcpt();
720	return False;
721
722	# ps fails with non-zero exit code if the pid wasn't found.
723	if iExitCode is not 0:
724	return False;
725	if sCurName is None:
726	return False;
727	sCurName = sCurName.strip();
728	if sCurName is '':
729	return False;
730
731	if os.path.basename(sName) == sName:
732	sCurName = os.path.basename(sCurName);
733	elif os.path.basename(sCurName) == sCurName:
734	sName = os.path.basename(sName);
735
736	if sCurName != sName:
737	return False;
738
739	fRc = True;
740	return fRc;
741
742
743	class ProcessInfo(object):
744	"""Process info."""
745	def __init__(self, iPid):
746	self.iPid = iPid;
747	self.iParentPid = None;
748	self.sImage = None;
749	self.sName = None;
750	self.asArgs = None;
751	self.sCwd = None;
752	self.iGid = None;
753	self.iUid = None;
754	self.iProcGroup = None;
755	self.iSessionId = None;
756
757	def loadAll(self):
758	"""Load all the info."""
759	sOs = getHostOs();
760	if sOs == 'linux':
761	sProc = '/proc/%s/' % (self.iPid,);
762	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
763	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
764	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
765	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
766	# sProc = '/proc/%s/' % (self.iPid,);
767	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
768	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
769	else:
770	pass;
771	if self.sName is None and self.sImage is not None:
772	self.sName = self.sImage;
773
774	def windowsGrabProcessInfo(self, oProcess):
775	"""Windows specific loadAll."""
776	try: self.sName = oProcess.Properties_("Name").Value;
777	except: pass;
778	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
779	except: pass;
780	try: self.asArgs = oProcess.Properties_("CommandLine").Value; ## @todo split it.
781	except: pass;
782	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
783	except: pass;
784	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
785	except: pass;
786	if self.sName is None and self.sImage is not None:
787	self.sName = self.sImage;
788
789	def getBaseImageName(self):
790	"""
791	Gets the base image name if available, use the process name if not available.
792	Returns image/process base name or None.
793	"""
794	sRet = self.sImage if self.sName is None else self.sName;
795	if sRet is None:
796	self.loadAll();
797	sRet = self.sImage if self.sName is None else self.sName;
798	if sRet is None:
799	if self.asArgs is None or len(self.asArgs) == 0:
800	return None;
801	sRet = self.asArgs[0];
802	if len(sRet) == 0:
803	return None;
804	return os.path.basename(sRet);
805
806	def getBaseImageNameNoExeSuff(self):
807	"""
808	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
809	"""
810	sRet = self.getBaseImageName();
811	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
812	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
813	sRet = sRet[:-4];
814	return sRet;
815
816
817	def processListAll(): # pylint: disable=R0914
818	"""
819	Return a list of ProcessInfo objects for all the processes in the system
820	that the current user can see.
821	"""
822	asProcesses = [];
823
824	sOs = getHostOs();
825	if sOs == 'win':
826	from win32com.client import GetObject; # pylint: disable=F0401
827	oWmi = GetObject('winmgmts:');
828	aoProcesses = oWmi.InstancesOf('Win32_Process');
829	for oProcess in aoProcesses:
830	try:
831	iPid = int(oProcess.Properties_("ProcessId").Value);
832	except:
833	continue;
834	oMyInfo = ProcessInfo(iPid);
835	oMyInfo.windowsGrabProcessInfo(oProcess);
836	asProcesses.append(oMyInfo);
837
838	elif sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
839	try:
840	asDirs = os.listdir('/proc');
841	except:
842	asDirs = [];
843	for sDir in asDirs:
844	if sDir.isdigit():
845	asProcesses.append(ProcessInfo(int(sDir),));
846
847	elif sOs == 'darwin':
848	# Try our best to parse ps output. (Not perfect but does the job most of the time.)
849	try:
850	sRaw = processOutputChecked([ '/bin/ps', '-A',
851	'-o', 'pid=',
852	'-o', 'ppid=',
853	'-o', 'pgid=',
854	'-o', 'sess=',
855	'-o', 'uid=',
856	'-o', 'gid=',
857	'-o', 'comm=' ]);
858	except:
859	return asProcesses;
860	elif sOs == 'solaris':
861	# Try our best to parse ps output. (Not perfect but does the job most of the time.)
862	try:
863	sRaw = processOutputChecked([ '/usr/ccs/bin/ps', '-A',
864	'-o', 'pid=',
865	'-o', 'ppid=',
866	'-o', 'pgid=',
867	'-o', 'sid=',
868	'-o', 'uid=',
869	'-o', 'gid=',
870	'-o', 'comm=' ]);
871	except:
872	return asProcesses;
873
874	for sLine in sRaw.split('\n'):
875	sLine = sLine.lstrip();
876	if len(sLine) < 7 or not sLine[0].isdigit():
877	continue;
878
879	iField = 0;
880	off = 0;
881	aoFields = [None, None, None, None, None, None, None];
882	while iField < 7:
883	# Eat whitespace.
884	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
885	off += 1;
886
887	# Final field / EOL.
888	if iField == 6:
889	aoFields[6] = sLine[off:];
890	break;
891	if off >= len(sLine):
892	break;
893
894	# Generic field parsing.
895	offStart = off;
896	off += 1;
897	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
898	off += 1;
899	try:
900	if iField != 3:
901	aoFields[iField] = int(sLine[offStart:off]);
902	else:
903	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
904	except:
905	pass;
906	iField += 1;
907
908	if aoFields[0] is not None:
909	oMyInfo = ProcessInfo(aoFields[0]);
910	oMyInfo.iParentPid = aoFields[1];
911	oMyInfo.iProcGroup = aoFields[2];
912	oMyInfo.iSessionId = aoFields[3];
913	oMyInfo.iUid = aoFields[4];
914	oMyInfo.iGid = aoFields[5];
915	oMyInfo.sName = aoFields[6];
916	asProcesses.append(oMyInfo);
917
918	return asProcesses;
919
920
921	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
922	"""
923	Looks for information regarding the demise of the given process.
924	"""
925	sOs = getHostOs();
926	if sOs == 'darwin':
927	#
928	# On darwin we look for crash and diagnostic reports.
929	#
930	asLogDirs = [
931	u'/Library/Logs/DiagnosticReports/',
932	u'/Library/Logs/CrashReporter/',
933	u'~/Library/Logs/DiagnosticReports/',
934	u'~/Library/Logs/CrashReporter/',
935	];
936	for sDir in asLogDirs:
937	sDir = os.path.expanduser(sDir);
938	if not os.path.isdir(sDir):
939	continue;
940	try:
941	asDirEntries = os.listdir(sDir);
942	except:
943	continue;
944	for sEntry in asDirEntries:
945	# Only interested in .crash files.
946	_, sSuff = os.path.splitext(sEntry);
947	if sSuff != '.crash':
948	continue;
949
950	# The pid can be found at the end of the first line.
951	sFull = os.path.join(sDir, sEntry);
952	try:
953	oFile = open(sFull, 'r');
954	sFirstLine = oFile.readline();
955	oFile.close();
956	except:
957	continue;
958	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
959	continue;
960	offPid = len(sFirstLine) - 3;
961	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
962	offPid -= 1;
963	try: uReportPid = int(sFirstLine[offPid:-2]);
964	except: continue;
965
966	# Does the pid we found match?
967	if uReportPid == uPid:
968	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
969	fnCrashFile(sFull, False);
970	elif sOs == 'win':
971	#
972	# Getting WER reports would be great, however we have trouble match the
973	# PID to those as they seems not to mention it in the brief reports.
974	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
975	# location - see the windows readme for the testbox script) and what
976	# the MSDN article lists for now.
977	#
978	# It's been observed on Windows server 2012 that the dump files takes
979	# the form: <processimage>.<decimal-pid>.dmp
980	#
981	asDmpDirs = [
982	u'%SystemDrive%/CrashDumps/', # Testboxes.
983	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
984	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
985	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
986	u'%WINDIR%/ServiceProfiles/',
987	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
988	];
989	sMatchSuffix = '.%u.dmp' % (uPid,);
990
991	for sDir in asDmpDirs:
992	sDir = os.path.expandvars(sDir);
993	if not os.path.isdir(sDir):
994	continue;
995	try:
996	asDirEntries = os.listdir(sDir);
997	except:
998	continue;
999	for sEntry in asDirEntries:
1000	if sEntry.endswith(sMatchSuffix):
1001	sFull = os.path.join(sDir, sEntry);
1002	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1003	fnCrashFile(sFull, True);
1004
1005	else:
1006	pass; ## TODO
1007	return None;
1008
1009
1010	#
1011	# Time.
1012	#
1013
1014	#
1015	# The following test case shows how time.time() only have ~ms resolution
1016	# on Windows (tested W10) and why it therefore makes sense to try use
1017	# performance counters.
1018	#
1019	# Note! We cannot use time.clock() as the timestamp must be portable across
1020	# processes. See timeout testcase problem on win hosts (no logs).
1021	#
1022	#import sys;
1023	#import time;
1024	#from common import utils;
1025	#
1026	#atSeries = [];
1027	#for i in xrange(1,160):
1028	# if i == 159: time.sleep(10);
1029	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1030	#
1031	#tPrev = atSeries[0]
1032	#for tCur in atSeries:
1033	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1034	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1035	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1036	# print '';
1037	# tPrev = tCur
1038	#
1039	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1040	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1041	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1042
1043	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1044	g_fpWinPerfCounterFreq = None;
1045	g_oFuncwinQueryPerformanceCounter = None;
1046
1047	def _winInitPerfCounter():
1048	""" Initializes the use of performance counters. """
1049	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1050
1051	uFrequency = ctypes.c_ulonglong(0);
1052	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1053	if uFrequency.value >= 1000:
1054	#print 'uFrequency = %s' % (uFrequency,);
1055	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1056	g_fpWinPerfCounterFreq = float(uFrequency.value);
1057
1058	# Check that querying the counter works too.
1059	global g_oFuncwinQueryPerformanceCounter
1060	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1061	uCurValue = ctypes.c_ulonglong(0);
1062	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1063	if uCurValue.value > 0:
1064	return True;
1065	g_fWinUseWinPerfCounter = False;
1066	return False;
1067
1068	def _winFloatTime():
1069	""" Gets floating point time on windows. """
1070	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1071	uCurValue = ctypes.c_ulonglong(0);
1072	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1073	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1074	return time.time();
1075
1076
1077	def timestampNano():
1078	"""
1079	Gets a nanosecond timestamp.
1080	"""
1081	if g_fWinUseWinPerfCounter is True:
1082	return long(_winFloatTime() * 1000000000);
1083	return long(time.time() * 1000000000);
1084
1085	def timestampMilli():
1086	"""
1087	Gets a millisecond timestamp.
1088	"""
1089	if g_fWinUseWinPerfCounter is True:
1090	return long(_winFloatTime() * 1000);
1091	return long(time.time() * 1000);
1092
1093	def timestampSecond():
1094	"""
1095	Gets a second timestamp.
1096	"""
1097	if g_fWinUseWinPerfCounter is True:
1098	return long(_winFloatTime());
1099	return long(time.time());
1100
1101	def getTimePrefix():
1102	"""
1103	Returns a timestamp prefix, typically used for logging. UTC.
1104	"""
1105	try:
1106	oNow = datetime.datetime.utcnow();
1107	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1108	except:
1109	sTs = 'getTimePrefix-exception';
1110	return sTs;
1111
1112	def getTimePrefixAndIsoTimestamp():
1113	"""
1114	Returns current UTC as log prefix and iso timestamp.
1115	"""
1116	try:
1117	oNow = datetime.datetime.utcnow();
1118	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1119	sTsIso = formatIsoTimestamp(oNow);
1120	except:
1121	sTsPrf = sTsIso = 'getTimePrefix-exception';
1122	return (sTsPrf, sTsIso);
1123
1124	def formatIsoTimestamp(oNow):
1125	"""Formats the datetime object as an ISO timestamp."""
1126	assert oNow.tzinfo is None;
1127	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1128	return sTs;
1129
1130	def getIsoTimestamp():
1131	"""Returns the current UTC timestamp as a string."""
1132	return formatIsoTimestamp(datetime.datetime.utcnow());
1133
1134
1135	def getLocalHourOfWeek():
1136	""" Local hour of week (0 based). """
1137	oNow = datetime.datetime.now();
1138	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1139
1140
1141	def formatIntervalSeconds(cSeconds):
1142	""" Format a seconds interval into a nice 01h 00m 22s string """
1143	# Two simple special cases.
1144	if cSeconds < 60:
1145	return '%ss' % (cSeconds,);
1146	if cSeconds < 3600:
1147	cMins = cSeconds / 60;
1148	cSecs = cSeconds % 60;
1149	if cSecs == 0:
1150	return '%sm' % (cMins,);
1151	return '%sm %ss' % (cMins, cSecs,);
1152
1153	# Generic and a bit slower.
1154	cDays = cSeconds / 86400;
1155	cSeconds %= 86400;
1156	cHours = cSeconds / 3600;
1157	cSeconds %= 3600;
1158	cMins = cSeconds / 60;
1159	cSecs = cSeconds % 60;
1160	sRet = '';
1161	if cDays > 0:
1162	sRet = '%sd ' % (cDays,);
1163	if cHours > 0:
1164	sRet += '%sh ' % (cHours,);
1165	if cMins > 0:
1166	sRet += '%sm ' % (cMins,);
1167	if cSecs > 0:
1168	sRet += '%ss ' % (cSecs,);
1169	assert len(sRet) > 0; assert sRet[-1] == ' ';
1170	return sRet[:-1];
1171
1172	def formatIntervalSeconds2(oSeconds):
1173	"""
1174	Flexible input version of formatIntervalSeconds for use in WUI forms where
1175	data is usually already string form.
1176	"""
1177	if isinstance(oSeconds, int) or isinstance(oSeconds, long):
1178	return formatIntervalSeconds(oSeconds);
1179	if not isString(oSeconds):
1180	try:
1181	lSeconds = long(oSeconds);
1182	except:
1183	pass;
1184	else:
1185	if lSeconds >= 0:
1186	return formatIntervalSeconds2(lSeconds);
1187	return oSeconds;
1188
1189	def parseIntervalSeconds(sString):
1190	"""
1191	Reverse of formatIntervalSeconds.
1192
1193	Returns (cSeconds, sError), where sError is None on success.
1194	"""
1195
1196	# We might given non-strings, just return them without any fuss.
1197	if not isString(sString):
1198	if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1199	return (sString, None);
1200	## @todo time/date objects?
1201	return (int(sString), None);
1202
1203	# Strip it and make sure it's not empty.
1204	sString = sString.strip();
1205	if len(sString) == 0:
1206	return (0, 'Empty interval string.');
1207
1208	#
1209	# Split up the input into a list of 'valueN, unitN, ...'.
1210	#
1211	# Don't want to spend too much time trying to make re.split do exactly what
1212	# I need here, so please forgive the extra pass I'm making here.
1213	#
1214	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1215	asParts = [];
1216	for sPart in asRawParts:
1217	sPart = sPart.strip();
1218	if len(sPart) > 0:
1219	asParts.append(sPart);
1220	if len(asParts) == 0:
1221	return (0, 'Empty interval string or something?');
1222
1223	#
1224	# Process them one or two at the time.
1225	#
1226	cSeconds = 0;
1227	asErrors = [];
1228	i = 0;
1229	while i < len(asParts):
1230	sNumber = asParts[i];
1231	i += 1;
1232	if sNumber.isdigit():
1233	iNumber = int(sNumber);
1234
1235	sUnit = 's';
1236	if i < len(asParts) and not asParts[i].isdigit():
1237	sUnit = asParts[i];
1238	i += 1;
1239
1240	sUnitLower = sUnit.lower();
1241	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1242	pass;
1243	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1244	iNumber *= 60;
1245	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1246	iNumber *= 3600;
1247	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1248	iNumber *= 86400;
1249	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1250	iNumber = 7 86400;
1251	else:
1252	asErrors.append('Unknown unit "%s".' % (sUnit,));
1253	cSeconds += iNumber;
1254	else:
1255	asErrors.append('Bad number "%s".' % (sNumber,));
1256	return (cSeconds, None if len(asErrors) == 0 else ' '.join(asErrors));
1257
1258	def formatIntervalHours(cHours):
1259	""" Format a hours interval into a nice 1w 2d 1h string. """
1260	# Simple special cases.
1261	if cHours < 24:
1262	return '%sh' % (cHours,);
1263
1264	# Generic and a bit slower.
1265	cWeeks = cHours / (7 * 24);
1266	cHours %= 7 * 24;
1267	cDays = cHours / 24;
1268	cHours %= 24;
1269	sRet = '';
1270	if cWeeks > 0:
1271	sRet = '%sw ' % (cWeeks,);
1272	if cDays > 0:
1273	sRet = '%sd ' % (cDays,);
1274	if cHours > 0:
1275	sRet += '%sh ' % (cHours,);
1276	assert len(sRet) > 0; assert sRet[-1] == ' ';
1277	return sRet[:-1];
1278
1279	def parseIntervalHours(sString):
1280	"""
1281	Reverse of formatIntervalHours.
1282
1283	Returns (cHours, sError), where sError is None on success.
1284	"""
1285
1286	# We might given non-strings, just return them without any fuss.
1287	if not isString(sString):
1288	if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1289	return (sString, None);
1290	## @todo time/date objects?
1291	return (int(sString), None);
1292
1293	# Strip it and make sure it's not empty.
1294	sString = sString.strip();
1295	if len(sString) == 0:
1296	return (0, 'Empty interval string.');
1297
1298	#
1299	# Split up the input into a list of 'valueN, unitN, ...'.
1300	#
1301	# Don't want to spend too much time trying to make re.split do exactly what
1302	# I need here, so please forgive the extra pass I'm making here.
1303	#
1304	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1305	asParts = [];
1306	for sPart in asRawParts:
1307	sPart = sPart.strip();
1308	if len(sPart) > 0:
1309	asParts.append(sPart);
1310	if len(asParts) == 0:
1311	return (0, 'Empty interval string or something?');
1312
1313	#
1314	# Process them one or two at the time.
1315	#
1316	cHours = 0;
1317	asErrors = [];
1318	i = 0;
1319	while i < len(asParts):
1320	sNumber = asParts[i];
1321	i += 1;
1322	if sNumber.isdigit():
1323	iNumber = int(sNumber);
1324
1325	sUnit = 'h';
1326	if i < len(asParts) and not asParts[i].isdigit():
1327	sUnit = asParts[i];
1328	i += 1;
1329
1330	sUnitLower = sUnit.lower();
1331	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1332	pass;
1333	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1334	iNumber *= 24;
1335	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1336	iNumber = 7 24;
1337	else:
1338	asErrors.append('Unknown unit "%s".' % (sUnit,));
1339	cHours += iNumber;
1340	else:
1341	asErrors.append('Bad number "%s".' % (sNumber,));
1342	return (cHours, None if len(asErrors) == 0 else ' '.join(asErrors));
1343
1344
1345	#
1346	# Introspection.
1347	#
1348
1349	def getCallerName(oFrame=None, iFrame=2):
1350	"""
1351	Returns the name of the caller's caller.
1352	"""
1353	if oFrame is None:
1354	try:
1355	raise Exception();
1356	except:
1357	oFrame = sys.exc_info()[2].tb_frame.f_back;
1358	while iFrame > 1:
1359	if oFrame is not None:
1360	oFrame = oFrame.f_back;
1361	iFrame = iFrame - 1;
1362	if oFrame is not None:
1363	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1364	return sName;
1365	return "unknown";
1366
1367
1368	def getXcptInfo(cFrames = 1):
1369	"""
1370	Gets text detailing the exception. (Good for logging.)
1371	Returns list of info strings.
1372	"""
1373
1374	#
1375	# Try get exception info.
1376	#
1377	try:
1378	oType, oValue, oTraceback = sys.exc_info();
1379	except:
1380	oType = oValue = oTraceback = None;
1381	if oType is not None:
1382
1383	#
1384	# Try format the info
1385	#
1386	asRet = [];
1387	try:
1388	try:
1389	asRet = asRet + traceback.format_exception_only(oType, oValue);
1390	asTraceBack = traceback.format_tb(oTraceback);
1391	if cFrames is not None and cFrames <= 1:
1392	asRet.append(asTraceBack[-1]);
1393	else:
1394	asRet.append('Traceback:')
1395	for iFrame in range(min(cFrames, len(asTraceBack))):
1396	asRet.append(asTraceBack[-iFrame - 1]);
1397	asRet.append('Stack:')
1398	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1399	except:
1400	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1401
1402	if len(asRet) == 0:
1403	asRet.append('No exception info...');
1404	except:
1405	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1406	else:
1407	asRet = ['Couldn\'t find exception traceback.'];
1408	return asRet;
1409
1410
1411	#
1412	# TestSuite stuff.
1413	#
1414
1415	def isRunningFromCheckout(cScriptDepth = 1):
1416	"""
1417	Checks if we're running from the SVN checkout or not.
1418	"""
1419
1420	try:
1421	sFile = __file__;
1422	cScriptDepth = 1;
1423	except:
1424	sFile = sys.argv[0];
1425
1426	sDir = os.path.abspath(sFile);
1427	while cScriptDepth >= 0:
1428	sDir = os.path.dirname(sDir);
1429	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1430	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1431	return True;
1432	cScriptDepth -= 1;
1433
1434	return False;
1435
1436
1437	#
1438	# Bourne shell argument fun.
1439	#
1440
1441
1442	def argsSplit(sCmdLine):
1443	"""
1444	Given a bourne shell command line invocation, split it up into arguments
1445	assuming IFS is space.
1446	Returns None on syntax error.
1447	"""
1448	## @todo bourne shell argument parsing!
1449	return sCmdLine.split(' ');
1450
1451	def argsGetFirst(sCmdLine):
1452	"""
1453	Given a bourne shell command line invocation, get return the first argument
1454	assuming IFS is space.
1455	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1456	"""
1457	asArgs = argsSplit(sCmdLine);
1458	if asArgs is None or len(asArgs) == 0:
1459	return None;
1460
1461	return asArgs[0];
1462
1463	#
1464	# String helpers.
1465	#
1466
1467	def stricmp(sFirst, sSecond):
1468	"""
1469	Compares to strings in an case insensitive fashion.
1470
1471	Python doesn't seem to have any way of doing the correctly, so this is just
1472	an approximation using lower.
1473	"""
1474	if sFirst == sSecond:
1475	return 0;
1476	sLower1 = sFirst.lower();
1477	sLower2 = sSecond.lower();
1478	if sLower1 == sLower2:
1479	return 0;
1480	if sLower1 < sLower2:
1481	return -1;
1482	return 1;
1483
1484
1485	#
1486	# Misc.
1487	#
1488
1489	def versionCompare(sVer1, sVer2):
1490	"""
1491	Compares to version strings in a fashion similar to RTStrVersionCompare.
1492	"""
1493
1494	## @todo implement me!!
1495
1496	if sVer1 == sVer2:
1497	return 0;
1498	if sVer1 < sVer2:
1499	return -1;
1500	return 1;
1501
1502
1503	def formatNumber(lNum, sThousandSep = ' '):
1504	"""
1505	Formats a decimal number with pretty separators.
1506	"""
1507	sNum = str(lNum);
1508	sRet = sNum[-3:];
1509	off = len(sNum) - 3;
1510	while off > 0:
1511	off -= 3;
1512	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
1513	return sRet;
1514
1515
1516	def formatNumberNbsp(lNum):
1517	"""
1518	Formats a decimal number with pretty separators.
1519	"""
1520	sRet = formatNumber(lNum);
1521	return unicode(sRet).replace(' ', u'\u00a0');
1522
1523
1524	def isString(oString):
1525	"""
1526	Checks if the object is a string object, hiding difference between python 2 and 3.
1527
1528	Returns True if it's a string of some kind.
1529	Returns False if not.
1530	"""
1531	if sys.version_info[0] >= 3:
1532	return isinstance(oString, str);
1533	return isinstance(oString, basestring);
1534
1535
1536	def hasNonAsciiCharacters(sText):
1537	"""
1538	Returns True is specified string has non-ASCII characters, False if ASCII only.
1539	"""
1540	sTmp = unicode(sText, errors='ignore') if isinstance(sText, str) else sText;
1541	return not all(ord(ch) < 128 for ch in sTmp);
1542
1543
1544	def chmodPlusX(sFile):
1545	"""
1546	Makes the specified file or directory executable.
1547	Returns success indicator, no exceptions.
1548
1549	Note! Symbolic links are followed and the target will be changed.
1550	"""
1551	try:
1552	oStat = os.stat(sFile);
1553	except:
1554	return False;
1555	try:
1556	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
1557	except:
1558	return False;
1559	return True;
1560
1561
1562	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1563	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1564	"""
1565	Worker for unpackFile that deals with ZIP files, same function signature.
1566	"""
1567	import zipfile
1568	if fnError is None:
1569	fnError = fnLog;
1570
1571	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
1572
1573	# Open it.
1574	try: oZipFile = zipfile.ZipFile(sArchive, 'r')
1575	except Exception as oXcpt:
1576	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1577	return None;
1578
1579	# Extract all members.
1580	asMembers = [];
1581	try:
1582	for sMember in oZipFile.namelist():
1583	if fnFilter is None or fnFilter(sMember) is not False:
1584	if sMember.endswith('/'):
1585	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
1586	else:
1587	oZipFile.extract(sMember, sDstDir);
1588	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
1589	except Exception as oXcpt:
1590	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1591	asMembers = None;
1592
1593	# close it.
1594	try: oZipFile.close();
1595	except Exception as oXcpt:
1596	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1597	asMembers = None;
1598
1599	return asMembers;
1600
1601
1602	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
1603	g_fTarCopyFileObjOverriddend = False;
1604
1605	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError):
1606	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
1607	if length is None:
1608	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
1609	elif length > 0:
1610	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
1611	for _ in xrange(cFull):
1612	abBuffer = src.read(g_cbGoodBufferSize);
1613	dst.write(abBuffer);
1614	if len(abBuffer) != g_cbGoodBufferSize:
1615	raise exception('unexpected end of source file');
1616	if cbRemainder > 0:
1617	abBuffer = src.read(cbRemainder);
1618	dst.write(abBuffer);
1619	if len(abBuffer) != cbRemainder:
1620	raise exception('unexpected end of source file');
1621
1622
1623	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1624	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1625	"""
1626	Worker for unpackFile that deals with tarballs, same function signature.
1627	"""
1628	import shutil;
1629	import tarfile;
1630	if fnError is None:
1631	fnError = fnLog;
1632
1633	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
1634
1635	#
1636	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
1637	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
1638	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
1639	#
1640	if True is True:
1641	__installShUtilHacks(shutil);
1642	global g_fTarCopyFileObjOverriddend;
1643	if g_fTarCopyFileObjOverriddend is False:
1644	g_fTarCopyFileObjOverriddend = True;
1645	tarfile.copyfileobj = __mytarfilecopyfileobj;
1646
1647	#
1648	# Open it.
1649	#
1650	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
1651	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
1652	#
1653	try: oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize);
1654	except Exception as oXcpt:
1655	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1656	return None;
1657
1658	# Extract all members.
1659	asMembers = [];
1660	try:
1661	for oTarInfo in oTarFile:
1662	try:
1663	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
1664	if oTarInfo.islnk():
1665	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
1666	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
1667	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
1668	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
1669	sParentDir = os.path.dirname(sLinkFile);
1670	try: os.unlink(sLinkFile);
1671	except: pass;
1672	if sParentDir is not '' and not os.path.exists(sParentDir):
1673	os.makedirs(sParentDir);
1674	try: os.link(sLinkTarget, sLinkFile);
1675	except: shutil.copy2(sLinkTarget, sLinkFile);
1676	else:
1677	if oTarInfo.isdir():
1678	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
1679	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
1680	oTarFile.extract(oTarInfo, sDstDir);
1681	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
1682	except Exception as oXcpt:
1683	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
1684	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
1685	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
1686	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
1687	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
1688	asMembers = None;
1689	break;
1690	except Exception as oXcpt:
1691	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1692	asMembers = None;
1693
1694	#
1695	# Finally, close it.
1696	#
1697	try: oTarFile.close();
1698	except Exception as oXcpt:
1699	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1700	asMembers = None;
1701
1702	return asMembers;
1703
1704
1705	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1706	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1707	"""
1708	Unpacks the given file if it has a know archive extension, otherwise do
1709	nothing.
1710
1711	fnLog & fnError both take a string parameter.
1712
1713	fnFilter takes a member name (string) and returns True if it's included
1714	and False if excluded.
1715
1716	Returns list of the extracted files (full path) on success.
1717	Returns empty list if not a supported archive format.
1718	Returns None on failure. Raises no exceptions.
1719	"""
1720	sBaseNameLower = os.path.basename(sArchive).lower();
1721
1722	#
1723	# Zip file?
1724	#
1725	if sBaseNameLower.endswith('.zip'):
1726	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1727
1728	#
1729	# Tarball?
1730	#
1731	if sBaseNameLower.endswith('.tar') \
1732	or sBaseNameLower.endswith('.tar.gz') \
1733	or sBaseNameLower.endswith('.tgz') \
1734	or sBaseNameLower.endswith('.tar.bz2'):
1735	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1736
1737	#
1738	# Cannot classify it from the name, so just return that to the caller.
1739	#
1740	fnLog('Not unpacking "%s".' % (sArchive,));
1741	return [];
1742
1743
1744	def getDiskUsage(sPath):
1745	"""
1746	Get free space of a partition that corresponds to specified sPath in MB.
1747
1748	Returns partition free space value in MB.
1749	"""
1750	if platform.system() == 'Windows':
1751	oCTypeFreeSpace = ctypes.c_ulonglong(0);
1752	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
1753	ctypes.pointer(oCTypeFreeSpace));
1754	cbFreeSpace = oCTypeFreeSpace.value;
1755	else:
1756	oStats = os.statvfs(sPath); # pylint: disable=E1101
1757	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
1758
1759	# Convert to MB
1760	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
1761
1762	return cMbFreeSpace;
1763
1764
1765	#
1766	# Unit testing.
1767	#
1768
1769	# pylint: disable=C0111
1770	class BuildCategoryDataTestCase(unittest.TestCase):
1771	def testIntervalSeconds(self):
1772	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
1773	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
1774	self.assertEqual(parseIntervalSeconds('123'), (123, None));
1775	self.assertEqual(parseIntervalSeconds(123), (123, None));
1776	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
1777	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
1778	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
1779	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
1780	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
1781	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
1782	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
1783
1784	def testHasNonAsciiChars(self):
1785	self.assertEqual(hasNonAsciiCharacters(''), False);
1786	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
1787	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
1788	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
1789	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
1790	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
1791
1792	if __name__ == '__main__':
1793	unittest.main();
1794	# not reached.
1795

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 65006

Download in other formats: