wangtao
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Analyzer Tests] Run static analyzer integration tests until the end, 

Do not stop at the first failure.

Differential Revision: https://reviews.llvm.org/D38589

llvm-svn: 314992
This commit is contained in:
George Karpenkov 2017-10-05 17:32:06 +00:00
parent 9f3e88ae64
commit 1b51cbddf4
1 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
clang/utils/analyzer/SATestBuild.py
View File

@ -570,7 +570,11 @@ def runCmpResults(Dir, Strictness=0):
0 - success if there are no crashes or analyzer failure. 0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs. 1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical. 2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
""" """
TestsPassed = True
TBegin = time.time() TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
@ -586,8 +590,6 @@ def runCmpResults(Dir, Strictness=0):
RefList.remove(RefLogDir) RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName)) NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) == 0 or len(NewList) == 0:
return False
assert(len(RefList) == len(NewList)) assert(len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build # There might be more then one folder underneath - one per each scan-build
@ -624,15 +626,15 @@ def runCmpResults(Dir, Strictness=0):
(NumDiffs, DiffsPath,) (NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0: if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)." print "Error: Diffs found in strict mode (2)."
sys.exit(-1) TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew: elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in "\ print "Error: The number of results are different in "\
"strict mode (1)." "strict mode (1)."
sys.exit(-1) TestsPassed = False
print "Diagnostic comparison complete (time: %.2f)." % ( print "Diagnostic comparison complete (time: %.2f)." % (
time.time() - TBegin) time.time() - TBegin)
return (NumDiffs > 0) return TestsPassed
def cleanupReferenceResults(SBOutputDir): def cleanupReferenceResults(SBOutputDir):
@ -686,6 +688,11 @@ def updateSVN(Mode, PMapFile):
def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
"""
Test a given project.
:return TestsPassed: Whether tests have passed according
to the :param Strictness: criteria.
"""
print " \n\n--- Building project %s" % (ID,) print " \n\n--- Building project %s" % (ID,)
TBegin = time.time() TBegin = time.time()
@ -704,11 +711,13 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
if IsReferenceBuild: if IsReferenceBuild:
cleanupReferenceResults(SBOutputDir) cleanupReferenceResults(SBOutputDir)
TestsPassed = True
else: else:
runCmpResults(Dir, Strictness) TestsPassed = runCmpResults(Dir, Strictness)
print "Completed tests for project %s (time: %.2f)." % \ print "Completed tests for project %s (time: %.2f)." % \
(ID, (time.time() - TBegin)) (ID, (time.time() - TBegin))
return TestsPassed
def isCommentCSVLine(Entries): def isCommentCSVLine(Entries):
@ -749,6 +758,7 @@ def validateProjectFile(PMapFile):
def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0): def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
TestsPassed = True
with projectFileHandler() as PMapFile: with projectFileHandler() as PMapFile:
validateProjectFile(PMapFile) validateProjectFile(PMapFile)
@ -760,7 +770,7 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
# Test the projects. # Test the projects.
for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile): for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
testProject( TestsPassed &= testProject(
ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness) ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)
# Re-add reference results to SVN. # Re-add reference results to SVN.
@ -793,4 +803,6 @@ if __name__ == '__main__':
IsReference = True IsReference = True
UpdateSVN = True UpdateSVN = True
testAll(IsReference, UpdateSVN, Strictness) TestsPassed = testAll(IsReference, UpdateSVN, Strictness)
if not TestsPassed:
sys.exit(-1)