298 lines
9.6 KiB
Python
298 lines
9.6 KiB
Python
import errno
|
|
import itertools
|
|
import math
|
|
import os
|
|
import platform
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
def to_bytes(str):
|
|
# Encode to UTF-8 to get binary data.
|
|
if isinstance(str, bytes):
|
|
return str
|
|
return str.encode('utf-8')
|
|
|
|
def to_string(bytes):
|
|
if isinstance(bytes, str):
|
|
return bytes
|
|
return to_bytes(bytes)
|
|
|
|
def convert_string(bytes):
|
|
try:
|
|
return to_string(bytes.decode('utf-8'))
|
|
except AttributeError: # 'str' object has no attribute 'decode'.
|
|
return str(bytes)
|
|
except UnicodeError:
|
|
return str(bytes)
|
|
|
|
def detectCPUs():
|
|
"""
|
|
Detects the number of CPUs on a system. Cribbed from pp.
|
|
"""
|
|
# Linux, Unix and MacOS:
|
|
if hasattr(os, "sysconf"):
|
|
if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
|
|
# Linux & Unix:
|
|
ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
|
|
if isinstance(ncpus, int) and ncpus > 0:
|
|
return ncpus
|
|
else: # OSX:
|
|
return int(capture(['sysctl', '-n', 'hw.ncpu']))
|
|
# Windows:
|
|
if "NUMBER_OF_PROCESSORS" in os.environ:
|
|
ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
|
|
if ncpus > 0:
|
|
# With more than 32 processes, process creation often fails with
|
|
# "Too many open files". FIXME: Check if there's a better fix.
|
|
return min(ncpus, 32)
|
|
return 1 # Default
|
|
|
|
def mkdir_p(path):
|
|
"""mkdir_p(path) - Make the "path" directory, if it does not exist; this
|
|
will also make directories for any missing parent directories."""
|
|
if not path or os.path.exists(path):
|
|
return
|
|
|
|
parent = os.path.dirname(path)
|
|
if parent != path:
|
|
mkdir_p(parent)
|
|
|
|
try:
|
|
os.mkdir(path)
|
|
except OSError:
|
|
e = sys.exc_info()[1]
|
|
# Ignore EEXIST, which may occur during a race condition.
|
|
if e.errno != errno.EEXIST:
|
|
raise
|
|
|
|
def capture(args, env=None):
|
|
"""capture(command) - Run the given command (or argv list) in a shell and
|
|
return the standard output. Raises a CalledProcessError if the command
|
|
exits with a non-zero status."""
|
|
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env=env)
|
|
out, err = p.communicate()
|
|
out = convert_string(out)
|
|
err = convert_string(err)
|
|
if p.returncode != 0:
|
|
raise subprocess.CalledProcessError(cmd=args,
|
|
returncode=p.returncode,
|
|
output="{}\n{}".format(out, err))
|
|
return out
|
|
|
|
def which(command, paths = None):
|
|
"""which(command, [paths]) - Look up the given command in the paths string
|
|
(or the PATH environment variable, if unspecified)."""
|
|
|
|
if paths is None:
|
|
paths = os.environ.get('PATH','')
|
|
|
|
# Check for absolute match first.
|
|
if os.path.isfile(command):
|
|
return command
|
|
|
|
# Would be nice if Python had a lib function for this.
|
|
if not paths:
|
|
paths = os.defpath
|
|
|
|
# Get suffixes to search.
|
|
# On Cygwin, 'PATHEXT' may exist but it should not be used.
|
|
if os.pathsep == ';':
|
|
pathext = os.environ.get('PATHEXT', '').split(';')
|
|
else:
|
|
pathext = ['']
|
|
|
|
# Search the paths...
|
|
for path in paths.split(os.pathsep):
|
|
for ext in pathext:
|
|
p = os.path.join(path, command + ext)
|
|
if os.path.exists(p) and not os.path.isdir(p):
|
|
return p
|
|
|
|
return None
|
|
|
|
def checkToolsPath(dir, tools):
|
|
for tool in tools:
|
|
if not os.path.exists(os.path.join(dir, tool)):
|
|
return False
|
|
return True
|
|
|
|
def whichTools(tools, paths):
|
|
for path in paths.split(os.pathsep):
|
|
if checkToolsPath(path, tools):
|
|
return path
|
|
return None
|
|
|
|
def printHistogram(items, title = 'Items'):
|
|
items.sort(key = lambda item: item[1])
|
|
|
|
maxValue = max([v for _,v in items])
|
|
|
|
# Select first "nice" bar height that produces more than 10 bars.
|
|
power = int(math.ceil(math.log(maxValue, 10)))
|
|
for inc in itertools.cycle((5, 2, 2.5, 1)):
|
|
barH = inc * 10**power
|
|
N = int(math.ceil(maxValue / barH))
|
|
if N > 10:
|
|
break
|
|
elif inc == 1:
|
|
power -= 1
|
|
|
|
histo = [set() for i in range(N)]
|
|
for name,v in items:
|
|
bin = min(int(N * v/maxValue), N-1)
|
|
histo[bin].add(name)
|
|
|
|
barW = 40
|
|
hr = '-' * (barW + 34)
|
|
print('\nSlowest %s:' % title)
|
|
print(hr)
|
|
for name,value in items[-20:]:
|
|
print('%.2fs: %s' % (value, name))
|
|
print('\n%s Times:' % title)
|
|
print(hr)
|
|
pDigits = int(math.ceil(math.log(maxValue, 10)))
|
|
pfDigits = max(0, 3-pDigits)
|
|
if pfDigits:
|
|
pDigits += pfDigits + 1
|
|
cDigits = int(math.ceil(math.log(len(items), 10)))
|
|
print("[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3),
|
|
'Percentage'.center(barW),
|
|
'Count'.center(cDigits*2 + 1)))
|
|
print(hr)
|
|
for i,row in enumerate(histo):
|
|
pct = float(len(row)) / len(items)
|
|
w = int(barW * pct)
|
|
print("[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" % (
|
|
pDigits, pfDigits, i*barH, pDigits, pfDigits, (i+1)*barH,
|
|
'*'*w, ' '*(barW-w), cDigits, len(row), cDigits, len(items)))
|
|
|
|
class ExecuteCommandTimeoutException(Exception):
|
|
def __init__(self, msg, out, err, exitCode):
|
|
assert isinstance(msg, str)
|
|
assert isinstance(out, str)
|
|
assert isinstance(err, str)
|
|
assert isinstance(exitCode, int)
|
|
self.msg = msg
|
|
self.out = out
|
|
self.err = err
|
|
self.exitCode = exitCode
|
|
|
|
# Close extra file handles on UNIX (on Windows this cannot be done while
|
|
# also redirecting input).
|
|
kUseCloseFDs = not (platform.system() == 'Windows')
|
|
def executeCommand(command, cwd=None, env=None, input=None, timeout=0):
|
|
"""
|
|
Execute command ``command`` (list of arguments or string)
|
|
with
|
|
* working directory ``cwd`` (str), use None to use the current
|
|
working directory
|
|
* environment ``env`` (dict), use None for none
|
|
* Input to the command ``input`` (str), use string to pass
|
|
no input.
|
|
* Max execution time ``timeout`` (int) seconds. Use 0 for no timeout.
|
|
|
|
Returns a tuple (out, err, exitCode) where
|
|
* ``out`` (str) is the standard output of running the command
|
|
* ``err`` (str) is the standard error of running the command
|
|
* ``exitCode`` (int) is the exitCode of running the command
|
|
|
|
If the timeout is hit an ``ExecuteCommandTimeoutException``
|
|
is raised.
|
|
"""
|
|
if input is not None:
|
|
input = to_bytes(input)
|
|
p = subprocess.Popen(command, cwd=cwd,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=env, close_fds=kUseCloseFDs)
|
|
timerObject = None
|
|
# FIXME: Because of the way nested function scopes work in Python 2.x we
|
|
# need to use a reference to a mutable object rather than a plain
|
|
# bool. In Python 3 we could use the "nonlocal" keyword but we need
|
|
# to support Python 2 as well.
|
|
hitTimeOut = [False]
|
|
try:
|
|
if timeout > 0:
|
|
def killProcess():
|
|
# We may be invoking a shell so we need to kill the
|
|
# process and all its children.
|
|
hitTimeOut[0] = True
|
|
killProcessAndChildren(p.pid)
|
|
|
|
timerObject = threading.Timer(timeout, killProcess)
|
|
timerObject.start()
|
|
|
|
out,err = p.communicate(input=input)
|
|
exitCode = p.wait()
|
|
finally:
|
|
if timerObject != None:
|
|
timerObject.cancel()
|
|
|
|
# Ensure the resulting output is always of string type.
|
|
out = convert_string(out)
|
|
err = convert_string(err)
|
|
|
|
if hitTimeOut[0]:
|
|
raise ExecuteCommandTimeoutException(
|
|
msg='Reached timeout of {} seconds'.format(timeout),
|
|
out=out,
|
|
err=err,
|
|
exitCode=exitCode
|
|
)
|
|
|
|
# Detect Ctrl-C in subprocess.
|
|
if exitCode == -signal.SIGINT:
|
|
raise KeyboardInterrupt
|
|
|
|
return out, err, exitCode
|
|
|
|
def usePlatformSdkOnDarwin(config, lit_config):
|
|
# On Darwin, support relocatable SDKs by providing Clang with a
|
|
# default system root path.
|
|
if 'darwin' in config.target_triple:
|
|
try:
|
|
cmd = subprocess.Popen(['xcrun', '--show-sdk-path', '--sdk', 'macosx'],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
out, err = cmd.communicate()
|
|
out = out.strip()
|
|
res = cmd.wait()
|
|
except OSError:
|
|
res = -1
|
|
if res == 0 and out:
|
|
sdk_path = out
|
|
lit_config.note('using SDKROOT: %r' % sdk_path)
|
|
config.environment['SDKROOT'] = sdk_path
|
|
|
|
def killProcessAndChildren(pid):
|
|
"""
|
|
This function kills a process with ``pid`` and all its
|
|
running children (recursively). It is currently implemented
|
|
using the psutil module which provides a simple platform
|
|
neutral implementation.
|
|
|
|
TODO: Reimplement this without using psutil so we can
|
|
remove our dependency on it.
|
|
"""
|
|
import psutil
|
|
try:
|
|
psutilProc = psutil.Process(pid)
|
|
# Handle the different psutil API versions
|
|
try:
|
|
# psutil >= 2.x
|
|
children_iterator = psutilProc.children(recursive=True)
|
|
except AttributeError:
|
|
# psutil 1.x
|
|
children_iterator = psutilProc.get_children(recursive=True)
|
|
for child in children_iterator:
|
|
try:
|
|
child.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
psutilProc.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|