Re: [edk2-devel] [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section
Reviewed-by: Bob Feng
-Original Message-
From: Rodriguez, Christian
Sent: Thursday, May 30, 2019 12:27 AM
To: devel@edk2.groups.io
Cc: Feng, Bob C ; Gao, Liming ;
Zhu, Yonghong
Subject: [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for
Sources section
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804
After adding a check to see if [Sources] section lists all the header type
files of a module, track module and library hashes for --hash feature. If above
check is not in compilance for a library or module, force hash invalidation on
that library or module.
Signed-off-by: Christian Rodriguez
Cc: Bob Feng
Cc: Liming Gao
Cc: Yonghong Zhu
---
BaseTools/Source/Python/AutoGen/AutoGen.py | 6 +-
BaseTools/Source/Python/AutoGen/GenMake.py | 6 ++
BaseTools/Source/Python/Common/GlobalData.py | 3 +-
BaseTools/Source/Python/build/build.py | 65
4 files changed, 53 insertions(+), 27 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
b/BaseTools/Source/Python/AutoGen/AutoGen.py
index a5bef4f7c6..a376bc24d6 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateMakeFile()
-if self.CanSkip():
+# Don't enable if hash feature enabled, CanSkip uses timestamps to
determine build skipping
+if not GlobalData.gUseHashCache and self.CanSkip():
return
if len(self.CustomMakefile) == 0:
@@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateCodeFile()
-if self.CanSkip():
+# Don't enable if hash feature enabled, CanSkip uses timestamps to
determine build skipping
+if not GlobalData.gUseHashCache and self.CanSkip():
return
AutoGenList = []
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 5c992d7c26..212ca0fa7f 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -935,10 +935,16 @@ cleanlib:
continue
headerFileDependencySet.add(aFileName)
+# Ensure that gModuleBuildTracking has been initialized per
architecture
+if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking:
+GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] =
+ dict()
+
# Check if a module dependency header file is missing from the
module's MetaFile
for aFile in headerFileDependencySet:
if aFile in headerFilesInMetaFileSet:
continue
+if GlobalData.gUseHashCache:
+
GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGenObject]
= 'FAIL_METAFILE'
EdkLogger.warn("build","Module MetaFile [Sources] is missing local
header!",
ExtraData = "Local Header: " + aFile + " not found in
" + self._AutoGenObject.MetaFile.Path
)
diff --git a/BaseTools/Source/Python/Common/GlobalData.py
b/BaseTools/Source/Python/Common/GlobalData.py
index 95e28a988f..bd45a43728 100644
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure -# False
-> Fail : True -> Success
+# Top Dict: Key: Arch Type Value: Dictionary
+# Second Dict: Key: AutoGen ObjValue: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
gModuleBuildTracking = dict()
# Dictionary of booleans that dictate whether a module or diff --git
a/BaseTools/Source/Python/build/build.py
b/BaseTools/Source/Python/build/build.py
index 80ceb98310..0855d4561c 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -625,8 +625,16 @@ class BuildTask:
BuildTask._ErrorFlag.set()
BuildTask._ErrorMessage = "%s broken\n%s [%s]" % \
(threading.currentThread().getName(),
Command, WorkingDir)
-if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and
not BuildTask._ErrorFlag.isSet():
-GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True
+
+# Set the value used by hash invalidation flow in
GlobalData.gModuleBuildTracking to 'SUCCESS'
+# If Module or Lib is being tracked, it did not fail header check
test, and built successfully
+if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking
and
+ self.BuildItem.BuildObject in
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and
+
[edk2-devel] [Patch V4 2/2] BaseTools: Refactor hash tracking after checking for Sources section
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1804
After adding a check to see if [Sources] section lists all the header
type files of a module, track module and library hashes for --hash
feature. If above check is not in compilance for a library or module,
force hash invalidation on that library or module.
Signed-off-by: Christian Rodriguez
Cc: Bob Feng
Cc: Liming Gao
Cc: Yonghong Zhu
---
BaseTools/Source/Python/AutoGen/AutoGen.py | 6 +-
BaseTools/Source/Python/AutoGen/GenMake.py | 6 ++
BaseTools/Source/Python/Common/GlobalData.py | 3 +-
BaseTools/Source/Python/build/build.py | 65
4 files changed, 53 insertions(+), 27 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGen.py
b/BaseTools/Source/Python/AutoGen/AutoGen.py
index a5bef4f7c6..a376bc24d6 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGen.py
@@ -3989,7 +3989,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateMakeFile()
-if self.CanSkip():
+# Don't enable if hash feature enabled, CanSkip uses timestamps to
determine build skipping
+if not GlobalData.gUseHashCache and self.CanSkip():
return
if len(self.CustomMakefile) == 0:
@@ -4032,7 +4033,8 @@ class ModuleAutoGen(AutoGen):
for LibraryAutoGen in self.LibraryAutoGenList:
LibraryAutoGen.CreateCodeFile()
-if self.CanSkip():
+# Don't enable if hash feature enabled, CanSkip uses timestamps to
determine build skipping
+if not GlobalData.gUseHashCache and self.CanSkip():
return
AutoGenList = []
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py
b/BaseTools/Source/Python/AutoGen/GenMake.py
index 5c992d7c26..212ca0fa7f 100644
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -935,10 +935,16 @@ cleanlib:
continue
headerFileDependencySet.add(aFileName)
+# Ensure that gModuleBuildTracking has been initialized per
architecture
+if self._AutoGenObject.Arch not in GlobalData.gModuleBuildTracking:
+GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch] = dict()
+
# Check if a module dependency header file is missing from the
module's MetaFile
for aFile in headerFileDependencySet:
if aFile in headerFilesInMetaFileSet:
continue
+if GlobalData.gUseHashCache:
+
GlobalData.gModuleBuildTracking[self._AutoGenObject.Arch][self._AutoGenObject]
= 'FAIL_METAFILE'
EdkLogger.warn("build","Module MetaFile [Sources] is missing local
header!",
ExtraData = "Local Header: " + aFile + " not found in
" + self._AutoGenObject.MetaFile.Path
)
diff --git a/BaseTools/Source/Python/Common/GlobalData.py
b/BaseTools/Source/Python/Common/GlobalData.py
index 95e28a988f..bd45a43728 100644
--- a/BaseTools/Source/Python/Common/GlobalData.py
+++ b/BaseTools/Source/Python/Common/GlobalData.py
@@ -110,7 +110,8 @@ gEnableGenfdsMultiThread = False
gSikpAutoGenCache = set()
# Dictionary for tracking Module build status as success or failure
-# False -> Fail : True -> Success
+# Top Dict: Key: Arch Type Value: Dictionary
+# Second Dict: Key: AutoGen ObjValue: 'SUCCESS'\'FAIL'\'FAIL_METAFILE'
gModuleBuildTracking = dict()
# Dictionary of booleans that dictate whether a module or
diff --git a/BaseTools/Source/Python/build/build.py
b/BaseTools/Source/Python/build/build.py
index 80ceb98310..0855d4561c 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -625,8 +625,16 @@ class BuildTask:
BuildTask._ErrorFlag.set()
BuildTask._ErrorMessage = "%s broken\n%s [%s]" % \
(threading.currentThread().getName(),
Command, WorkingDir)
-if self.BuildItem.BuildObject in GlobalData.gModuleBuildTracking and
not BuildTask._ErrorFlag.isSet():
-GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject] = True
+
+# Set the value used by hash invalidation flow in
GlobalData.gModuleBuildTracking to 'SUCCESS'
+# If Module or Lib is being tracked, it did not fail header check
test, and built successfully
+if (self.BuildItem.BuildObject.Arch in GlobalData.gModuleBuildTracking
and
+ self.BuildItem.BuildObject in
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch] and
+
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject]
!= 'FAIL_METAFILE' and
+ not BuildTask._ErrorFlag.isSet()
+ ):
+
GlobalData.gModuleBuildTracking[self.BuildItem.BuildObject.Arch][self.BuildItem.BuildObject]
= 'SUCCESS'
+
