Re: [edk2-devel] [PATCH] BaseTools: Hash false success with back to back builds
The patch looks good for me. Reviewed-by: Bob Feng -Bob -Original Message- From: Rodriguez, Christian Sent: Wednesday, April 10, 2019 11:27 PM To: devel@edk2.groups.io Cc: Feng, Bob C ; Gao, Liming ; Zhu, Yonghong Subject: [PATCH] BaseTools: Hash false success with back to back builds BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1692 Add error handling to the --hash feature so that hash files are invalidated when a build error occurs. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/Common/GlobalData.py | 4 BaseTools/Source/Python/build/build.py | 45 + 2 files changed, 49 insertions(+) diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py index 1853f1d2f6..79f23c892d 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -108,3 +108,7 @@ gPackageHash = {} gModuleHash = {} gEnableGenfdsMultiThread = False gSikpAutoGenCache = set() + +# Dictionary for tracking Module build status as success or failure # +False -> Fail : True -> Success gModuleBuildTracking = dict() diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index af8bba47b1..71478b7268 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -620,6 +620,8 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage = "%s broken\n%s [%s]" % \ (threading.currentThread().getName(), Command, WorkingDir) +if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet(): +GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] + = True # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -1138,6 +1140,37 @@ class Build(): if Process.returncode != 0 : EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!') EdkLogger.info("\n- Postbuild Done -\n") + +## Error handling for hash feature +# +# On BuildTask error, iterate through the Module Build tracking +# dictionary to determine wheather a module failed to build. Invalidate +# the hash associated with that module by removing it from storage. +# +# +def invalidateHash(self): +# GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash +for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): +# False == FAIL : True == Success +# Skip invalidating for Successful module builds +if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True: +continue + +# The module failed to build or failed to start building, + from this point on + +# Remove .hash from build +if GlobalData.gUseHashCache: +ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash") +if os.path.exists(ModuleHashFile): +os.remove(ModuleHashFile) + +# Remove .hash file from cache +if GlobalData.gBinCacheSource: +FileDir = path.join(GlobalData.gBinCacheSource, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) +HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash') +if os.path.exists(HashFile): +os.remove(HashFile) + ## Build a module or platform # # Create autogen code and makefile for a module or platform, and the launch @@ -1795,6 +1828,9 @@ class Build(): if self.Target == "genmake": return True self.BuildModules.append(Ma) +# Initialize all modules in tracking to False (FAIL) +if Ma not in GlobalData.gModuleBuildTracking: +GlobalData.gModuleBuildTracking[Ma] = + False self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: @@ -1805,6 +1841,7 @@ class Build(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() +self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
[edk2-devel] [PATCH] BaseTools: Hash false success with back to back builds
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1692 Add error handling to the --hash feature so that hash files are invalidated when a build error occurs. Signed-off-by: Christian Rodriguez Cc: Bob Feng Cc: Liming Gao Cc: Yonghong Zhu --- BaseTools/Source/Python/Common/GlobalData.py | 4 BaseTools/Source/Python/build/build.py | 45 + 2 files changed, 49 insertions(+) diff --git a/BaseTools/Source/Python/Common/GlobalData.py b/BaseTools/Source/Python/Common/GlobalData.py index 1853f1d2f6..79f23c892d 100644 --- a/BaseTools/Source/Python/Common/GlobalData.py +++ b/BaseTools/Source/Python/Common/GlobalData.py @@ -108,3 +108,7 @@ gPackageHash = {} gModuleHash = {} gEnableGenfdsMultiThread = False gSikpAutoGenCache = set() + +# Dictionary for tracking Module build status as success or failure +# False -> Fail : True -> Success +gModuleBuildTracking = dict() diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py index af8bba47b1..71478b7268 100644 --- a/BaseTools/Source/Python/build/build.py +++ b/BaseTools/Source/Python/build/build.py @@ -620,6 +620,8 @@ class BuildTask: BuildTask._ErrorFlag.set() BuildTask._ErrorMessage = "%s broken\n%s [%s]" % \ (threading.currentThread().getName(), Command, WorkingDir) +if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and not BuildTask._ErrorFlag.isSet(): +GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True # indicate there's a thread is available for another build task BuildTask._RunningQueueLock.acquire() BuildTask._RunningQueue.pop(self.BuildItem) @@ -1138,6 +1140,37 @@ class Build(): if Process.returncode != 0 : EdkLogger.error("Postbuild", POSTBUILD_ERROR, 'Postbuild process is not success!') EdkLogger.info("\n- Postbuild Done -\n") + +## Error handling for hash feature +# +# On BuildTask error, iterate through the Module Build tracking +# dictionary to determine wheather a module failed to build. Invalidate +# the hash associated with that module by removing it from storage. +# +# +def invalidateHash(self): +# GlobalData.gModuleBuildTracking contains only modules that cannot be skipped by hash +for moduleAutoGenObj in GlobalData.gModuleBuildTracking.keys(): +# False == FAIL : True == Success +# Skip invalidating for Successful module builds +if GlobalData.gModuleBuildTracking[moduleAutoGenObj] == True: +continue + +# The module failed to build or failed to start building, from this point on + +# Remove .hash from build +if GlobalData.gUseHashCache: +ModuleHashFile = path.join(moduleAutoGenObj.BuildDir, moduleAutoGenObj.Name + ".hash") +if os.path.exists(ModuleHashFile): +os.remove(ModuleHashFile) + +# Remove .hash file from cache +if GlobalData.gBinCacheSource: +FileDir = path.join(GlobalData.gBinCacheSource, moduleAutoGenObj.Arch, moduleAutoGenObj.SourceDir, moduleAutoGenObj.MetaFile.BaseName) +HashFile = path.join(FileDir, moduleAutoGenObj.Name + '.hash') +if os.path.exists(HashFile): +os.remove(HashFile) + ## Build a module or platform # # Create autogen code and makefile for a module or platform, and the launch @@ -1795,6 +1828,9 @@ class Build(): if self.Target == "genmake": return True self.BuildModules.append(Ma) +# Initialize all modules in tracking to False (FAIL) +if Ma not in GlobalData.gModuleBuildTracking: +GlobalData.gModuleBuildTracking[Ma] = False self.AutoGenTime += int(round((time.time() - AutoGenStart))) MakeStart = time.time() for Ma in self.BuildModules: @@ -1805,6 +1841,7 @@ class Build(): # we need a full version of makefile for platform ExitFlag.set() BuildTask.WaitForComplete() +self.invalidateHash() Pa.CreateMakeFile(False) EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule) # Start task scheduler @@ -1814,6 +1851,7 @@ class Build(): # in case there's an interruption. we need a full version of makefile for platform Pa.CreateMakeFile(False) if BuildTask.HasError(): +self.inv