[gem5-dev] Change in gem5/gem5[develop]: scons: Pull some python related mechanisms out of USE_PYTHON guards.
Gabe Black has submitted this change. (
https://gem5-review.googlesource.com/c/public/gem5/+/48367 )
Change subject: scons: Pull some python related mechanisms out of
USE_PYTHON guards.
..
scons: Pull some python related mechanisms out of USE_PYTHON guards.
We don't want to build certain files if USE_PYTHON is disabled, but we
can still tell scons how to.
Change-Id: I38c7c93f609cfcedc350f8270f0b239b69c4f101
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/48367
Tested-by: kokoro
Reviewed-by: Bobby R. Bruce
Maintainer: Bobby R. Bruce
---
M SConstruct
M src/SConscript
2 files changed, 72 insertions(+), 73 deletions(-)
Approvals:
Bobby R. Bruce: Looks good to me, approved; Looks good to me, approved
kokoro: Regressions pass
diff --git a/SConstruct b/SConstruct
index 396dc59..4e2ed47 100755
--- a/SConstruct
+++ b/SConstruct
@@ -516,12 +516,6 @@
if not py_version:
error("Can't find a working Python installation")
-marshal_env = main.Clone()
-
-# Bare minimum environment that only includes python
-marshal_env.Append(CCFLAGS='$MARSHAL_CCFLAGS_EXTRA')
-marshal_env.Append(LINKFLAGS='$MARSHAL_LDFLAGS_EXTRA')
-
# Found a working Python installation. Check if it meets minimum
# requirements.
ver_string = '.'.join(map(str, py_version))
@@ -532,6 +526,12 @@
warning('Embedded python library too new. '
'Python 3 expected, found %s.' % ver_string)
+marshal_env = main.Clone()
+
+# Bare minimum environment that only includes python
+marshal_env.Append(CCFLAGS='$MARSHAL_CCFLAGS_EXTRA')
+marshal_env.Append(LINKFLAGS='$MARSHAL_LDFLAGS_EXTRA')
+
main['HAVE_PKG_CONFIG'] = main.Detect('pkg-config')
with gem5_scons.Configure(main) as conf:
@@ -704,9 +704,7 @@
env.Append(CCFLAGS='$CCFLAGS_EXTRA')
env.Append(LINKFLAGS='$LDFLAGS_EXTRA')
-exports=['env']
-if main['USE_PYTHON']:
-exports.append('marshal_env')
+exports=['env', 'marshal_env']
# The src/SConscript file sets up the build rules in 'env' according
# to the configured variables. It returns a list of environments,
diff --git a/src/SConscript b/src/SConscript
index db11f44..6fc3276 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -349,6 +349,67 @@
class Source(SourceFile):
pass
+# Build a small helper that marshals the Python code using the same version
+# of Python as gem5. This is in an unorthodox location to avoid building it
+# for every variant.
+py_marshal = marshal_env.Program('marshal', 'python/marshal.cc')[0]
+
+# Embed python files. All .py files that have been indicated by a
+# PySource() call in a SConscript need to be embedded into the M5
+# library. To do that, we compile the file to byte code, marshal the
+# byte code, compress it, and then generate a c++ file that
+# inserts the result into an array.
+def embedPyFile(target, source, env):
+def c_str(string):
+if string is None:
+return "0"
+return '"%s"' % string
+
+'''Action function to compile a .py into a code object, marshal it,
+compress it, and stick it into an asm file so the code appears as
+just bytes with a label in the data section. The action takes two
+sources:
+
+source[0]: Binary used to marshal Python sources
+source[1]: Python script to marshal
+'''
+
+import subprocess
+
+marshalled = subprocess.check_output(
+[source[0].abspath, str(source[1])], env=env['ENV'])
+
+compressed = zlib.compress(marshalled)
+data = compressed
+pysource = PySource.tnodes[source[1]]
+
+code = code_formatter()
+code('''\
+#include "sim/init.hh"
+
+namespace gem5
+{
+namespace
+{
+
+''')
+bytesToCppArray(code, 'embedded_module_data', data)
+# The name of the EmbeddedPython object doesn't matter since it's in an
+# anonymous namespace, and it's constructor takes care of installing it
+# into a global list.
+code('''
+EmbeddedPython embedded_module_info(
+${{c_str(pysource.abspath)}},
+${{c_str(pysource.modpath)}},
+embedded_module_data,
+${{len(data)}},
+${{len(marshalled)}});
+
+} // anonymous namespace
+} // namespace gem5
+''')
+code.write(str(target[0]))
+
class PySource(SourceFile):
'''Add a python source file to the named package'''
modules = {}
@@ -384,6 +445,9 @@
PySource.modules[modpath] = self
PySource.tnodes[self.tnode] = self
+marshal_env.Command(self.cpp, [ py_marshal, self.tnode ],
+MakeAction(embedPyFile, Transform("EMBED PY")))
+
class SimObject(PySource):
'''Add a SimObject python file as a python source object and add
it to a list of sim object modules'''
@@ -1037,7 +1101,7 @@
MakeAction(createSimObjectWrappers,
Transform("SO PyB/C")))
env.Depends(cc_file, depends + extra_deps)
-Source(cc_file)
[gem5-dev] Change in gem5/gem5[develop]: scons: Pull some python related mechanisms out of USE_PYTHON guards.
Gabe Black has uploaded this change for review. (
https://gem5-review.googlesource.com/c/public/gem5/+/48367 )
Change subject: scons: Pull some python related mechanisms out of
USE_PYTHON guards.
..
scons: Pull some python related mechanisms out of USE_PYTHON guards.
We don't want to build certain files if USE_PYTHON is disabled, but we
can still tell scons how to.
Change-Id: I38c7c93f609cfcedc350f8270f0b239b69c4f101
---
M SConstruct
M src/SConscript
2 files changed, 69 insertions(+), 70 deletions(-)
diff --git a/SConstruct b/SConstruct
index 2b2e39c..7deb6e4 100755
--- a/SConstruct
+++ b/SConstruct
@@ -512,12 +512,6 @@
if not py_version:
error("Can't find a working Python installation")
-marshal_env = main.Clone()
-
-# Bare minimum environment that only includes python
-marshal_env.Append(CCFLAGS='$MARSHAL_CCFLAGS_EXTRA')
-marshal_env.Append(LINKFLAGS='$MARSHAL_LDFLAGS_EXTRA')
-
# Found a working Python installation. Check if it meets minimum
# requirements.
ver_string = '.'.join(map(str, py_version))
@@ -528,6 +522,12 @@
warning('Embedded python library too new. '
'Python 3 expected, found %s.' % ver_string)
+marshal_env = main.Clone()
+
+# Bare minimum environment that only includes python
+marshal_env.Append(CCFLAGS='$MARSHAL_CCFLAGS_EXTRA')
+marshal_env.Append(LINKFLAGS='$MARSHAL_LDFLAGS_EXTRA')
+
main['HAVE_PKG_CONFIG'] = main.Detect('pkg-config')
with gem5_scons.Configure(main) as conf:
@@ -700,9 +700,7 @@
env.Append(CCFLAGS='$CCFLAGS_EXTRA')
env.Append(LINKFLAGS='$LDFLAGS_EXTRA')
-exports=['env']
-if main['USE_PYTHON']:
-exports.append('marshal_env')
+exports=['env', 'marshal_env']
# The src/SConscript file sets up the build rules in 'env' according
# to the configured variables. It returns a list of environments,
diff --git a/src/SConscript b/src/SConscript
index 589a0e4..7c81f30 100644
--- a/src/SConscript
+++ b/src/SConscript
@@ -347,6 +347,64 @@
class Source(SourceFile):
pass
+# Build a small helper that marshals the Python code using the same version
+# of Python as gem5. This is in an unorthodox location to avoid building it
+# for every variant.
+py_marshal = marshal_env.Program('marshal', 'python/marshal.cc')[0]
+
+# Embed python files. All .py files that have been indicated by a
+# PySource() call in a SConscript need to be embedded into the M5
+# library. To do that, we compile the file to byte code, marshal the
+# byte code, compress it, and then generate a c++ file that
+# inserts the result into an array.
+def embedPyFile(target, source, env):
+def c_str(string):
+if string is None:
+return "0"
+return '"%s"' % string
+
+'''Action function to compile a .py into a code object, marshal it,
+compress it, and stick it into an asm file so the code appears as
+just bytes with a label in the data section. The action takes two
+sources:
+
+source[0]: Binary used to marshal Python sources
+source[1]: Python script to marshal
+'''
+
+import subprocess
+
+marshalled = subprocess.check_output(
+[source[0].abspath, str(source[1])], env=env['ENV'])
+
+compressed = zlib.compress(marshalled)
+data = compressed
+pysource = PySource.tnodes[source[1]]
+
+code = code_formatter()
+code('''\
+#include "sim/init.hh"
+
+namespace gem5
+{
+namespace
+{
+
+''')
+bytesToCppArray(code, 'embedded_module_data', data)
+code('''
+EmbeddedPython embedded_module_info(
+${{c_str(pysource.abspath)}},
+${{c_str(pysource.modpath)}},
+embedded_module_data,
+${{len(data)}},
+${{len(marshalled)}});
+
+} // anonymous namespace
+} // namespace gem5
+''')
+code.write(str(target[0]))
+
class PySource(SourceFile):
'''Add a python source file to the named package'''
modules = {}
@@ -382,6 +440,9 @@
PySource.modules[modpath] = self
PySource.tnodes[self.tnode] = self
+marshal_env.Command(self.cpp, [ py_marshal, self.tnode ],
+MakeAction(embedPyFile, Transform("EMBED PY")))
+
class SimObject(PySource):
'''Add a SimObject python file as a python source object and add
it to a list of sim object modules'''
@@ -1088,7 +1149,7 @@
MakeAction(createSimObjectWrappers,
Transform("SO PyB/C")))
env.Depends(cc_file, depends + extra_deps)
-Source(cc_file)
+Source(cc_file, add_tags='python')
#
# Handle debug flags
@@ -1218,68 +1279,8 @@
Transform("VER TAGS")))
env.AlwaysBuild(tags)
-# Embed python files. All .py files that have been indicated by a
-# PySource() call in a SConscript need to be embedded into the M5
-# library. To do that, we compile the file to byte code, marshal the
-# byte code, compress it, and then generate a c++ file that
-#
