428 lines
14 KiB
Python
Executable File
428 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Given a previous good compile narrow down miscompiles.
|
|
# Expects two directories named "before" and "after" each containing a set of
|
|
# assembly or object files where the "after" version is assumed to be broken.
|
|
# You also have to provide a script called "link_test". It is called with a
|
|
# list of files which should be linked together and result tested. "link_test"
|
|
# should returns with exitcode 0 if the linking and testing succeeded.
|
|
#
|
|
# If a response file is provided, only the object files that are listed in the
|
|
# file are inspected. In addition, the "link_test" is called with a temporary
|
|
# response file representing one iteration of bisection.
|
|
#
|
|
# abtest.py operates by taking all files from the "before" directory and
|
|
# in each step replacing one of them with a file from the "bad" directory.
|
|
#
|
|
# Additionally you can perform the same steps with a single .s file. In this
|
|
# mode functions are identified by " -- Begin function FunctionName" and
|
|
# " -- End function" markers. The abtest.py then takes all
|
|
# function from the file in the "before" directory and replaces one function
|
|
# with the corresponding function from the "bad" file in each step.
|
|
#
|
|
# Example usage to identify miscompiled files:
|
|
# 1. Create a link_test script, make it executable. Simple Example:
|
|
# clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
|
|
# 2. Run the script to figure out which files are miscompiled:
|
|
# > ./abtest.py
|
|
# somefile.s: ok
|
|
# someotherfile.s: skipped: same content
|
|
# anotherfile.s: failed: './link_test' exitcode != 0
|
|
# ...
|
|
# Example usage to identify miscompiled functions inside a file:
|
|
# 3. Run the tests on a single file (assuming before/file.s and
|
|
# after/file.s exist)
|
|
# > ./abtest.py file.s
|
|
# funcname1 [0/XX]: ok
|
|
# funcname2 [1/XX]: ok
|
|
# funcname3 [2/XX]: skipped: same content
|
|
# funcname4 [3/XX]: failed: './link_test' exitcode != 0
|
|
# ...
|
|
from fnmatch import filter
|
|
from sys import stderr
|
|
import argparse
|
|
import filecmp
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
# Specify LINKTEST via `--test`. Default value is './link_test'.
|
|
LINKTEST = ""
|
|
ESCAPE = "\033[%sm"
|
|
BOLD = ESCAPE % "1"
|
|
RED = ESCAPE % "31"
|
|
NORMAL = ESCAPE % "0"
|
|
FAILED = RED + "failed" + NORMAL
|
|
|
|
|
|
def find(dir, file_filter=None):
|
|
files = [
|
|
walkdir[0]+"/"+file
|
|
for walkdir in os.walk(dir)
|
|
for file in walkdir[2]
|
|
]
|
|
if file_filter is not None:
|
|
files = filter(files, file_filter)
|
|
return sorted(files)
|
|
|
|
|
|
def error(message):
|
|
stderr.write("Error: %s\n" % (message,))
|
|
|
|
|
|
def warn(message):
|
|
stderr.write("Warning: %s\n" % (message,))
|
|
|
|
|
|
def info(message):
|
|
stderr.write("Info: %s\n" % (message,))
|
|
|
|
|
|
def announce_test(name):
|
|
stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
|
|
stderr.flush()
|
|
|
|
|
|
def announce_result(result):
|
|
stderr.write(result)
|
|
stderr.write("\n")
|
|
stderr.flush()
|
|
|
|
|
|
def format_namelist(l):
|
|
result = ", ".join(l[0:3])
|
|
if len(l) > 3:
|
|
result += "... (%d total)" % len(l)
|
|
return result
|
|
|
|
|
|
def check_sanity(choices, perform_test):
|
|
announce_test("sanity check A")
|
|
all_a = {name: a_b[0] for name, a_b in choices}
|
|
res_a = perform_test(all_a)
|
|
if res_a is not True:
|
|
error("Picking all choices from A failed to pass the test")
|
|
sys.exit(1)
|
|
|
|
announce_test("sanity check B (expecting failure)")
|
|
all_b = {name: a_b[1] for name, a_b in choices}
|
|
res_b = perform_test(all_b)
|
|
if res_b is not False:
|
|
error("Picking all choices from B did unexpectedly pass the test")
|
|
sys.exit(1)
|
|
|
|
|
|
def check_sequentially(choices, perform_test):
|
|
known_good = set()
|
|
all_a = {name: a_b[0] for name, a_b in choices}
|
|
n = 1
|
|
for name, a_b in sorted(choices):
|
|
picks = dict(all_a)
|
|
picks[name] = a_b[1]
|
|
announce_test("checking %s [%d/%d]" % (name, n, len(choices)))
|
|
n += 1
|
|
res = perform_test(picks)
|
|
if res is True:
|
|
known_good.add(name)
|
|
return known_good
|
|
|
|
|
|
def check_bisect(choices, perform_test):
|
|
known_good = set()
|
|
if len(choices) == 0:
|
|
return known_good
|
|
|
|
choice_map = dict(choices)
|
|
all_a = {name: a_b[0] for name, a_b in choices}
|
|
|
|
def test_partition(partition, upcoming_partition):
|
|
# Compute the maximum number of checks we have to do in the worst case.
|
|
max_remaining_steps = len(partition) * 2 - 1
|
|
if upcoming_partition is not None:
|
|
max_remaining_steps += len(upcoming_partition) * 2 - 1
|
|
for x in partitions_to_split:
|
|
max_remaining_steps += (len(x) - 1) * 2
|
|
|
|
picks = dict(all_a)
|
|
for x in partition:
|
|
picks[x] = choice_map[x][1]
|
|
announce_test("checking %s [<=%d remaining]" %
|
|
(format_namelist(partition), max_remaining_steps))
|
|
res = perform_test(picks)
|
|
if res is True:
|
|
known_good.update(partition)
|
|
elif len(partition) > 1:
|
|
partitions_to_split.insert(0, partition)
|
|
|
|
# TODO:
|
|
# - We could optimize based on the knowledge that when splitting a failed
|
|
# partition into two and one side checks out okay then we can deduce that
|
|
# the other partition must be a failure.
|
|
all_choice_names = [name for name, _ in choices]
|
|
partitions_to_split = [all_choice_names]
|
|
while len(partitions_to_split) > 0:
|
|
partition = partitions_to_split.pop()
|
|
|
|
middle = len(partition) // 2
|
|
left = partition[0:middle]
|
|
right = partition[middle:]
|
|
|
|
if len(left) > 0:
|
|
test_partition(left, right)
|
|
assert len(right) > 0
|
|
test_partition(right, None)
|
|
|
|
return known_good
|
|
|
|
|
|
def extract_functions(file):
|
|
functions = []
|
|
in_function = None
|
|
for line in open(file):
|
|
marker = line.find(" -- Begin function ")
|
|
if marker != -1:
|
|
if in_function is not None:
|
|
warn("Missing end of function %s" % (in_function,))
|
|
funcname = line[marker + 19:-1]
|
|
in_function = funcname
|
|
text = line
|
|
continue
|
|
|
|
marker = line.find(" -- End function")
|
|
if marker != -1:
|
|
text += line
|
|
functions.append((in_function, text))
|
|
in_function = None
|
|
continue
|
|
|
|
if in_function is not None:
|
|
text += line
|
|
return functions
|
|
|
|
|
|
def replace_functions(source, dest, replacements):
|
|
out = open(dest, "w")
|
|
skip = False
|
|
in_function = None
|
|
for line in open(source):
|
|
marker = line.find(" -- Begin function ")
|
|
if marker != -1:
|
|
if in_function is not None:
|
|
warn("Missing end of function %s" % (in_function,))
|
|
funcname = line[marker + 19:-1]
|
|
in_function = funcname
|
|
replacement = replacements.get(in_function)
|
|
if replacement is not None:
|
|
out.write(replacement)
|
|
skip = True
|
|
else:
|
|
marker = line.find(" -- End function")
|
|
if marker != -1:
|
|
in_function = None
|
|
if skip:
|
|
skip = False
|
|
continue
|
|
|
|
if not skip:
|
|
out.write(line)
|
|
|
|
|
|
def testrun(files):
|
|
linkline = "%s %s" % (LINKTEST, " ".join(files),)
|
|
res = subprocess.call(linkline, shell=True)
|
|
if res != 0:
|
|
announce_result(FAILED + ": '%s' exitcode != 0" % LINKTEST)
|
|
return False
|
|
else:
|
|
announce_result("ok")
|
|
return True
|
|
|
|
|
|
def prepare_files(gooddir, baddir, rspfile):
|
|
files_a = []
|
|
files_b = []
|
|
|
|
if rspfile is not None:
|
|
def get_basename(name):
|
|
# remove prefix
|
|
if name.startswith(gooddir):
|
|
return name[len(gooddir):]
|
|
if name.startswith(baddir):
|
|
return name[len(baddir):]
|
|
assert False, ""
|
|
|
|
with open(rspfile, "r") as rf:
|
|
for line in rf.read().splitlines():
|
|
for obj in line.split():
|
|
assert not os.path.isabs(obj), "TODO: support abs path"
|
|
files_a.append(gooddir + "/" + obj)
|
|
files_b.append(baddir + "/" + obj)
|
|
else:
|
|
get_basename = lambda name: os.path.basename(name)
|
|
files_a = find(gooddir, "*")
|
|
files_b = find(baddir, "*")
|
|
|
|
basenames_a = set(map(get_basename, files_a))
|
|
basenames_b = set(map(get_basename, files_b))
|
|
|
|
for name in files_b:
|
|
basename = get_basename(name)
|
|
if basename not in basenames_a:
|
|
warn("There is no corresponding file to '%s' in %s" %
|
|
(name, gooddir))
|
|
choices = []
|
|
skipped = []
|
|
for name in files_a:
|
|
basename = get_basename(name)
|
|
if basename not in basenames_b:
|
|
warn("There is no corresponding file to '%s' in %s" %
|
|
(name, baddir))
|
|
|
|
file_a = gooddir + "/" + basename
|
|
file_b = baddir + "/" + basename
|
|
if filecmp.cmp(file_a, file_b):
|
|
skipped.append(basename)
|
|
continue
|
|
|
|
choice = (basename, (file_a, file_b))
|
|
choices.append(choice)
|
|
|
|
if len(skipped) > 0:
|
|
info("Skipped (same content): %s" % format_namelist(skipped))
|
|
|
|
def perform_test(picks):
|
|
files = []
|
|
# Note that we iterate over files_a so we don't change the order
|
|
# (cannot use `picks` as it is a dictionary without order)
|
|
for x in files_a:
|
|
basename = get_basename(x)
|
|
picked = picks.get(basename)
|
|
if picked is None:
|
|
assert basename in skipped
|
|
files.append(x)
|
|
else:
|
|
files.append(picked)
|
|
|
|
# If response file is used, create a temporary response file for the
|
|
# picked files.
|
|
if rspfile is not None:
|
|
with tempfile.NamedTemporaryFile('w', suffix='.rsp',
|
|
delete=False) as tf:
|
|
tf.write(" ".join(files))
|
|
tf.flush()
|
|
ret = testrun([tf.name])
|
|
os.remove(tf.name)
|
|
return ret
|
|
|
|
return testrun(files)
|
|
|
|
return perform_test, choices
|
|
|
|
|
|
def prepare_functions(to_check, gooddir, goodfile, badfile):
|
|
files_good = find(gooddir, "*")
|
|
|
|
functions_a = extract_functions(goodfile)
|
|
functions_a_map = dict(functions_a)
|
|
functions_b_map = dict(extract_functions(badfile))
|
|
|
|
for name in functions_b_map.keys():
|
|
if name not in functions_a_map:
|
|
warn("Function '%s' missing from good file" % name)
|
|
choices = []
|
|
skipped = []
|
|
for name, candidate_a in functions_a:
|
|
candidate_b = functions_b_map.get(name)
|
|
if candidate_b is None:
|
|
warn("Function '%s' missing from bad file" % name)
|
|
continue
|
|
if candidate_a == candidate_b:
|
|
skipped.append(name)
|
|
continue
|
|
choice = name, (candidate_a, candidate_b)
|
|
choices.append(choice)
|
|
|
|
if len(skipped) > 0:
|
|
info("Skipped (same content): %s" % format_namelist(skipped))
|
|
|
|
combined_file = '/tmp/combined2.s'
|
|
files = []
|
|
found_good_file = False
|
|
for c in files_good:
|
|
if os.path.basename(c) == to_check:
|
|
found_good_file = True
|
|
files.append(combined_file)
|
|
continue
|
|
files.append(c)
|
|
assert found_good_file
|
|
|
|
def perform_test(picks):
|
|
for name, x in picks.items():
|
|
assert x == functions_a_map[name] or x == functions_b_map[name]
|
|
replace_functions(goodfile, combined_file, picks)
|
|
return testrun(files)
|
|
return perform_test, choices
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--a', dest='dir_a', default='before')
|
|
parser.add_argument('--b', dest='dir_b', default='after')
|
|
parser.add_argument('--rsp', default=None)
|
|
parser.add_argument('--test', default='./link_test')
|
|
parser.add_argument('--insane', help='Skip sanity check',
|
|
action='store_true')
|
|
parser.add_argument('--seq',
|
|
help='Check sequentially instead of bisection',
|
|
action='store_true')
|
|
parser.add_argument('file', metavar='file', nargs='?')
|
|
config = parser.parse_args()
|
|
|
|
gooddir = config.dir_a
|
|
baddir = config.dir_b
|
|
rspfile = config.rsp
|
|
global LINKTEST
|
|
LINKTEST = config.test
|
|
|
|
# Preparation phase: Creates a dictionary mapping names to a list of two
|
|
# choices each. The bisection algorithm will pick one choice for each name
|
|
# and then run the perform_test function on it.
|
|
if config.file is not None:
|
|
goodfile = gooddir + "/" + config.file
|
|
badfile = baddir + "/" + config.file
|
|
perform_test, choices = prepare_functions(config.file, gooddir,
|
|
goodfile, badfile)
|
|
else:
|
|
perform_test, choices = prepare_files(gooddir, baddir, rspfile)
|
|
|
|
info("%d bisection choices" % len(choices))
|
|
|
|
# "Checking whether build environment is sane ..."
|
|
if not config.insane:
|
|
if not os.access(LINKTEST, os.X_OK):
|
|
error("Expect '%s' to be present and executable" % (LINKTEST,))
|
|
exit(1)
|
|
|
|
check_sanity(choices, perform_test)
|
|
|
|
if config.seq:
|
|
known_good = check_sequentially(choices, perform_test)
|
|
else:
|
|
known_good = check_bisect(choices, perform_test)
|
|
|
|
stderr.write("")
|
|
if len(known_good) != len(choices):
|
|
stderr.write("== Failing ==\n")
|
|
for name, _ in choices:
|
|
if name not in known_good:
|
|
stderr.write("%s\n" % name)
|
|
else:
|
|
# This shouldn't happen when the sanity check works...
|
|
# Maybe link_test isn't deterministic?
|
|
stderr.write("Could not identify failing parts?!?")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|