Author: george.karpenkov Date: Thu Oct 5 10:32:06 2017 New Revision: 314992
URL: http://llvm.org/viewvc/llvm-project?rev=314992&view=rev Log: [Analyzer Tests] Run static analyzer integration tests until the end, Do not stop at the first failure. Differential Revision: https://reviews.llvm.org/D38589 Modified: cfe/trunk/utils/analyzer/SATestBuild.py Modified: cfe/trunk/utils/analyzer/SATestBuild.py URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/analyzer/SATestBuild.py?rev=314992&r1=314991&r2=314992&view=diff ============================================================================== --- cfe/trunk/utils/analyzer/SATestBuild.py (original) +++ cfe/trunk/utils/analyzer/SATestBuild.py Thu Oct 5 10:32:06 2017 @@ -570,7 +570,11 @@ def runCmpResults(Dir, Strictness=0): 0 - success if there are no crashes or analyzer failure. 1 - success if there are no difference in the number of reported bugs. 2 - success if all the bug reports are identical. + + :return success: Whether tests pass according to the Strictness + criteria. """ + TestsPassed = True TBegin = time.time() RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName) @@ -586,8 +590,6 @@ def runCmpResults(Dir, Strictness=0): RefList.remove(RefLogDir) NewList.remove(os.path.join(NewDir, LogFolderName)) - if len(RefList) == 0 or len(NewList) == 0: - return False assert(len(RefList) == len(NewList)) # There might be more then one folder underneath - one per each scan-build @@ -624,15 +626,15 @@ def runCmpResults(Dir, Strictness=0): (NumDiffs, DiffsPath,) if Strictness >= 2 and NumDiffs > 0: print "Error: Diffs found in strict mode (2)." - sys.exit(-1) + TestsPassed = False elif Strictness >= 1 and ReportsInRef != ReportsInNew: print "Error: The number of results are different in "\ "strict mode (1)." - sys.exit(-1) + TestsPassed = False print "Diagnostic comparison complete (time: %.2f)." % ( time.time() - TBegin) - return (NumDiffs > 0) + return TestsPassed def cleanupReferenceResults(SBOutputDir): @@ -686,6 +688,11 @@ def updateSVN(Mode, PMapFile): def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0): + """ + Test a given project. + :return TestsPassed: Whether tests have passed according + to the :param Strictness: criteria. + """ print " \n\n--- Building project %s" % (ID,) TBegin = time.time() @@ -704,11 +711,13 @@ def testProject(ID, ProjectBuildMode, Is if IsReferenceBuild: cleanupReferenceResults(SBOutputDir) + TestsPassed = True else: - runCmpResults(Dir, Strictness) + TestsPassed = runCmpResults(Dir, Strictness) print "Completed tests for project %s (time: %.2f)." % \ (ID, (time.time() - TBegin)) + return TestsPassed def isCommentCSVLine(Entries): @@ -749,6 +758,7 @@ def validateProjectFile(PMapFile): def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0): + TestsPassed = True with projectFileHandler() as PMapFile: validateProjectFile(PMapFile) @@ -760,7 +770,7 @@ def testAll(IsReferenceBuild=False, Upda # Test the projects. for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile): - testProject( + TestsPassed &= testProject( ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness) # Re-add reference results to SVN. @@ -793,4 +803,6 @@ if __name__ == '__main__': IsReference = True UpdateSVN = True - testAll(IsReference, UpdateSVN, Strictness) + TestsPassed = testAll(IsReference, UpdateSVN, Strictness) + if not TestsPassed: + sys.exit(-1) _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
