Diff
Modified: trunk/Tools/ChangeLog (279983 => 279984)
--- trunk/Tools/ChangeLog 2021-07-16 11:16:13 UTC (rev 279983)
+++ trunk/Tools/ChangeLog 2021-07-16 14:25:26 UTC (rev 279984)
@@ -1,3 +1,65 @@
+2021-07-16 Angelos Oikonomopoulos <ange...@igalia.com>
+
+ Bundle libraries for remote execution in run-jsc-benchmarks
+ https://bugs.webkit.org/show_bug.cgi?id=227579
+
+ Reviewed by Carlos Alberto Lopez Perez.
+
+ Similarly to run-jsc-stress-tests, introduce a bundling step that
+ ships all library dependencies to the remote system when using
+ run-jsc-benchmarks --remote.
+
+ This patch factors out the code to
+ - lookup the ELF interpreter/libraries and to
+ - strip the rpath and create a wrapper script
+ from generate-bundle and places it webkitpy/binary_bundling.
+
+ It also introduces a simpler script that only bundles a single
+ binary (bundle-binary) and switches run-jsc-benchmarks and
+ run-jsc-stress-tests to use it.
+
+ It also updates run-jsc-benchmark to propagate any environment
+ variables intended for consumption by the JSC binary.
+
+ * Scripts/generate-bundle: Factor out reusable bundling code.
+ * Scripts/run-jsc-benchmarks: Do bundling and variable propagation.
+ * Scripts/run-jsc-stress-tests: Switch to bundle-binary.
+ * Scripts/webkitpy/binary_bundling/__init__.py: Added.
+ * Scripts/webkitpy/binary_bundling/bundle.py: Added.
+ (BinaryBundler):
+ (BinaryBundler.__init__):
+ (BinaryBundler.destination_dir):
+ (BinaryBundler.copy_and_remove_rpath):
+ (BinaryBundler.generate_wrapper_script):
+ * Scripts/webkitpy/binary_bundling/ldd.py: Added.
+ (SharedObjectResolver):
+ (SharedObjectResolver.__init__):
+ (SharedObjectResolver._run_cmd_and_get_output):
+ (SharedObjectResolver._get_interpreter_objname):
+ (SharedObjectResolver._get_libs_and_interpreter):
+ (SharedObjectResolver._ldd_recursive_get_libs_and_interpreter):
+ (SharedObjectResolver.get_libs_and_interpreter):
+
+ * Scripts/bundle-binary: Added.
+ * Scripts/generate-bundle:
+ * Scripts/run-jsc-benchmarks:
+ * Scripts/run-jsc-stress-tests:
+ * Scripts/webkitpy/binary_bundling/__init__.py: Added.
+ * Scripts/webkitpy/binary_bundling/bundle.py: Added.
+ (BinaryBundler):
+ (BinaryBundler.__init__):
+ (BinaryBundler.destination_dir):
+ (BinaryBundler.copy_and_remove_rpath):
+ (BinaryBundler.generate_wrapper_script):
+ * Scripts/webkitpy/binary_bundling/ldd.py: Added.
+ (SharedObjectResolver):
+ (SharedObjectResolver.__init__):
+ (SharedObjectResolver._run_cmd_and_get_output):
+ (SharedObjectResolver._get_interpreter_objname):
+ (SharedObjectResolver._get_libs_and_interpreter):
+ (SharedObjectResolver._ldd_recursive_get_libs_and_interpreter):
+ (SharedObjectResolver.get_libs_and_interpreter):
+
2021-07-16 Philippe Normand <pnorm...@igalia.com>
[GStreamer] LibWebRTC files should be in libwebrtc/gstreamer
Added: trunk/Tools/Scripts/bundle-binary (0 => 279984)
--- trunk/Tools/Scripts/bundle-binary (rev 0)
+++ trunk/Tools/Scripts/bundle-binary 2021-07-16 14:25:26 UTC (rev 279984)
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Copyright (C) 2021 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import logging
+import os
+import sys
+
+top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
+sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'Scripts', 'webkitpy'))
+
+from binary_bundling.ldd import SharedObjectResolver
+from binary_bundling.bundle import BinaryBundler
+
+def main():
+ parser = argparse.ArgumentParser('usage %prog [options]')
+ parser.add_argument('--ldd', dest='ldd', default='ldd', help='Use alternative ldd (useful for non-native binaries')
+ parser.add_argument('--log-level', dest='log_level', choices=['debug', 'info', 'warning', 'error', 'critical'], default='info')
+ parser.add_argument('--dest-dir', dest='dest_dir', required=True)
+ parser.add_argument('binary')
+ args = parser.parse_args()
+
+ logging.getLogger().setLevel(args.log_level.upper())
+ logging.getLogger().addHandler(logging.StreamHandler())
+ libraries, interpreter = SharedObjectResolver(args.ldd).get_libs_and_interpreter(args.binary)
+
+ if interpreter is None:
+ raise RuntimeError("Could not determine interpreter for binary %s" % object)
+ bundler = BinaryBundler(args.dest_dir, False)
+ bundler.copy_and_remove_rpath(interpreter, type='interpreter')
+ bundler.copy_and_remove_rpath(args.binary, type='bin')
+ for lib in libraries:
+ bundler.copy_and_remove_rpath(lib, type='lib')
+ bundler.generate_wrapper_script(interpreter, os.path.basename(args.binary))
+ return 0
+
+if __name__ == '__main__':
+ sys.exit(main())
Property changes on: trunk/Tools/Scripts/bundle-binary
___________________________________________________________________
Added: svn:executable
+*
\ No newline at end of property
Modified: trunk/Tools/Scripts/generate-bundle (279983 => 279984)
--- trunk/Tools/Scripts/generate-bundle 2021-07-16 11:16:13 UTC (rev 279983)
+++ trunk/Tools/Scripts/generate-bundle 2021-07-16 14:25:26 UTC (rev 279984)
@@ -40,10 +40,12 @@
top_level_directory = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'flatpak'))
sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'jhbuild'))
+sys.path.insert(0, os.path.join(top_level_directory, 'Tools', 'Scripts', 'webkitpy'))
import jhbuildutils
import flatpakutils
+from binary_bundling.ldd import SharedObjectResolver
+from binary_bundling.bundle import BinaryBundler
-
INSTALL_DEPS_SCRIPT_TEMPLATE = """\
#!/bin/bash
set -eu -o pipefail
@@ -127,7 +129,7 @@
self._bundle_type = bundle_type
self._buildername = builder_name
self._syslibs = syslibs
- self._ldd = ldd
+ self._shared_object_resolver = SharedObjectResolver(ldd)
self._should_strip_objects = should_strip_objects
self._compression_type = compression_type
self._tmpdir = None
@@ -151,77 +153,16 @@
return tempfile.mkdtemp(prefix=os.path.join(os.path.abspath(basedir), 'tmp'))
return tempfile.mkdtemp()
+
def _run_cmd_and_get_output(self, command):
- command_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+ _log.debug("EXEC %s" % command)
+ command_process = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding='utf-8')
stdout, stderr = command_process.communicate()
return command_process.returncode, stdout, stderr
-
- def _get_interpreter_objname(self, object):
- # Note: we use patchelf to get the object name (not the path!)
- # of the interpreter because this works regardless of the
- # architecture of the ELF file.
- retcode, stdout, stderr = self._run_cmd_and_get_output(['patchelf', '--print-interpreter', object])
- if retcode != 0:
- _log.debug("patchelf stdout:\n%s\nPatchelf stderr:\n%s" % (stdout, stderr))
- if 'cannot find section' in stdout:
- # This is fine; we only expect an interpreter in the main binary.
- return None
- raise RuntimeError('The patchelf command returned non-zero status for object %s' % object)
- interpreter_path = PurePath(stdout.strip())
- return interpreter_path.name
-
- def _get_libs_and_interpreter(self, object):
- interpreter = None
- retcode, stdout, stderr = self._run_cmd_and_get_output([self._ldd, object])
- _log.debug("ldd stdout:\n%s" % stdout)
- if retcode != 0:
- raise RuntimeError('The %s command returned non-zero status for object %s' % (self._ldd, object))
- libs = []
- for line in stdout.splitlines():
- line = line.strip()
- if '=>' in line:
- line = line.split('=>')[1].strip()
- if 'not found' in line:
- raise RuntimeError('ldd can not resolve all dependencies for object %s.' % object)
- line = line.split(' ')[0].strip()
- if os.path.isfile(line):
- libs.append(line)
- else:
- line = line.split(' ')[0].strip()
- if os.path.isfile(line):
- interpreter = line
- if interpreter is None:
- # This is the case for non-native binaries. For those, we
- # can use a cross-ldd (xldd), but then the interpreter
- # looks like any other shared object in the output of
- # ldd. Try to identify it by looking at the object name
- # from the interpreter string.
- interpreter_objname = self._get_interpreter_objname(object)
- for lib in libs:
- if PurePath(lib).name == interpreter_objname:
- interpreter = lib
- break
- # If we found an interpreter, remove it from the libs.
- libs = [lib for lib in libs if lib != interpreter]
- return libs, interpreter
-
-
- def _ldd_recursive_get_libs_and_interpreter(self, object, already_checked_libs = []):
- libs, interpreter = self._get_libs_and_interpreter(object)
- if libs:
- for lib in libs:
- if lib in already_checked_libs:
- continue
- # avoid recursion loops (libfreetype.so.6 <-> libharfbuzz.so.0)
- already_checked_libs.append(lib)
- sub_libs, sub_interpreter = self._ldd_recursive_get_libs_and_interpreter(lib, already_checked_libs)
- libs.extend(sub_libs)
- if sub_interpreter and interpreter and sub_interpreter != interpreter:
- raise RuntimeError('library %s has interpreter %s but object %s has interpreter %s' % (lib, sub_interpreter, object, interpreter))
- return list(set(libs)), interpreter
-
-
def _get_osprettyname(self):
with open('/etc/os-release', 'r') as osrelease_handle:
for line in osrelease_handle.readlines():
@@ -258,37 +199,30 @@
def _generate_wrapper_script(self, interpreter, binary_to_wrap):
- if not os.path.isfile(os.path.join(self._tmpdir, 'bin', binary_to_wrap)):
- raise RuntimeError('Can not find binary to wrap for %s' % binary_to_wrap)
- self._wrapper_scripts.append(binary_to_wrap)
- _log.info('Generate wrapper script %s' % binary_to_wrap)
- script_file = os.path.join(self._tmpdir, binary_to_wrap)
+ variables = dict()
+ mydir = self._bundler.VAR_MYDIR
+ if os.path.isdir(os.path.join(self._bundler.destination_dir(), 'gio')):
+ gio_var = 'GIO_MODULE_DIR' if self._syslibs == 'bundle-all' else 'GIO_EXTRA_MODULES'
+ variables[gio_var] = "${%s}/gio" % mydir
- with open(script_file, 'w') as script_handle:
- script_handle.write('#!/bin/sh\n')
- script_handle.write('MYDIR="$(dirname $(readlink -f $0))"\n')
- script_handle.write('export LD_LIBRARY_PATH="${MYDIR}/lib"\n')
- if os.path.isdir(os.path.join(self._tmpdir, 'gio')):
- gio_var = 'GIO_MODULE_DIR' if self._syslibs == 'bundle-all' else 'GIO_EXTRA_MODULES'
- script_handle.write('export %s="${MYDIR}/gio"\n' % gio_var)
- if os.path.isdir(os.path.join(self._tmpdir, 'gst')):
- gst_var = 'GST_PLUGIN_SYSTEM_PATH_1_0' if self._syslibs == 'bundle-all' else 'GST_PLUGIN_PATH_1_0'
- script_handle.write('export %s="${MYDIR}/gst"\n' % gst_var)
- script_handle.write('export GST_REGISTRY_1_0="${MYDIR}/gst/gstreamer-1.0.registry"\n')
+ if os.path.isdir(os.path.join(self._bundler.destination_dir(), 'gst')):
+ gst_var = 'GST_PLUGIN_SYSTEM_PATH_1_0' if self._syslibs == 'bundle-all' else 'GST_PLUGIN_PATH_1_0'
+ variables[gst_var] = "${%s}/gst" % mydir
+ variables['GST_REGISTRY_1_0'] = "${%s}/gst/gstreamer-1.0.registry" % mydir
+ if binary_to_wrap != "jsc":
+ variables['WEBKIT_EXEC_PATH'] = "${%s}/bin" % mydir
+ variables['WEBKIT_INJECTED_BUNDLE_PATH'] = "${%s}/lib" % mydir
+ if self._syslibs == 'bundle-all':
if binary_to_wrap != "jsc":
- script_handle.write('export WEBKIT_EXEC_PATH="${MYDIR}/bin"\n')
- script_handle.write('export WEBKIT_INJECTED_BUNDLE_PATH="${MYDIR}/lib"\n')
- if self._syslibs == 'bundle-all':
- script_handle.write('INTERPRETER="${MYDIR}/lib/%s"\n' % os.path.basename(interpreter))
- if binary_to_wrap != "jsc":
- script_handle.write('export WEB_PROCESS_CMD_PREFIX="${INTERPRETER}"\n')
- script_handle.write('export PLUGIN_PROCESS_CMD_PREFIX="${INTERPRETER}"\n')
- script_handle.write('export NETWORK_PROCESS_CMD_PREFIX="${INTERPRETER}"\n')
- script_handle.write('export GPU_PROCESS_CMD_PREFIX="${INTERPRETER}"\n')
- script_handle.write('exec "${INTERPRETER}" "${MYDIR}/bin/%s" "$@"\n' % binary_to_wrap)
- else:
- script_handle.write('exec "${MYDIR}/bin/%s" "$@"\n' % binary_to_wrap)
- os.chmod(script_file, 0o755)
+ for var in ['WEB_PROCESS_CMD_PREFIX',
+ 'PLUGIN_PROCESS_CMD_PREFIX',
+ 'NETWORK_PROCESS_CMD_PREFIX',
+ 'GPU_PROCESS_CMD_PREFIX']:
+ variables[var] = "${%s}" % self._bundler.VAR_INTERPRETER
+ else:
+ interpreter = None
+ self._bundler.generate_wrapper_script(interpreter, binary_to_wrap, variables)
+ self._wrapper_scripts.append(binary_to_wrap)
def _generate_install_deps_script(self, system_packages_needed):
if not system_packages_needed:
@@ -312,35 +246,6 @@
installdeps_handle.write(INSTALL_DEPS_SCRIPT_TEMPLATE % {'packages_needed' : ' '.join(system_packages_needed)} )
os.chmod(installdeps_file, 0o755)
-
- def _copy_and_remove_rpath(self, orig_file, type='bin', destination_dir=None):
- if not destination_dir:
- dir_suffix = 'lib' if type == 'interpreter' else type
- destination_dir = os.path.join(self._tmpdir, dir_suffix)
- if not os.path.isdir(destination_dir):
- os.makedirs(destination_dir)
-
- if not os.path.isfile(orig_file):
- raise ValueError('Can not find file %s' % orig_file)
-
- _log.info('Add to bundle [%s]: %s' % (type, orig_file))
- shutil.copy(orig_file, destination_dir)
-
- if shutil.which('patchelf'):
- patch_elf_command = ['patchelf', '--remove-rpath', os.path.join(destination_dir, os.path.basename(orig_file))]
- if subprocess.call(patch_elf_command) != 0:
- _log.warning('The patchelf command returned non-zero status')
- else:
- _log.warning('patchelf not found. Not modifying rpath')
-
- if self._should_strip_objects:
- if shutil.which('strip'):
- strip_command = ['strip', '--strip-unneeded', os.path.join(destination_dir, os.path.basename(orig_file))]
- if subprocess.call(strip_command) != 0:
- _log.warning('The strip command returned non-zero status')
- else:
- _log.warning('strip not found. Not stripping object')
-
def _remove_tempdir(self):
if not self._tmpdir:
return
@@ -349,6 +254,7 @@
def create(self):
self._tmpdir = self._create_tempdir(self._buildpath)
+ self._bundler = BinaryBundler(self._tmpdir, self._should_strip_objects)
if os.path.isfile(self._bundle_file_path):
_log.info('Removing previous bundle %s' % self._bundle_file_path)
@@ -456,11 +362,11 @@
def _add_object_or_get_sysdep(self, object, object_type):
provided_by_system_package = None
if self._syslibs == 'bundle-all':
- self._copy_and_remove_rpath(object, type=object_type)
+ self._bundler.copy_and_remove_rpath(object, type=object_type)
else:
provided_by_system_package = self._get_system_package_name(object)
if not provided_by_system_package:
- self._copy_and_remove_rpath(object, type=object_type)
+ self._bundler.copy_and_remove_rpath(object, type=object_type)
return provided_by_system_package
def _ensure_wpe_backend_symlink(self):
@@ -506,20 +412,20 @@
if system_package:
system_packages_needed.add(system_package)
elif object.endswith('.so'):
- self._copy_and_remove_rpath(object, type='lib')
+ self._bundler.copy_and_remove_rpath(object, type='lib')
else:
- self._copy_and_remove_rpath(object, type='bin')
+ self._bundler.copy_and_remove_rpath(object, type='bin')
# There is no need to examine the libraries linked with objects coming from a system package,
# because system packages already declare dependencies between them.
# However, if we are running with self._syslibs == 'bundle-all' then system_package will be None,
# and everything will be examined and bundled as we don't account for system packages in that case.
if not system_package:
- libraries, interpreter = self._ldd_recursive_get_libs_and_interpreter(object)
+ libraries, interpreter = self._shared_object_resolver.get_libs_and_interpreter(object)
if interpreter is None:
raise RuntimeError("Could not determine interpreter for binary %s" % object)
if copied_interpreter is None:
if self._syslibs == 'bundle-all':
- self._copy_and_remove_rpath(interpreter, type='interpreter')
+ self._bundler.copy_and_remove_rpath(interpreter, type='interpreter')
copied_interpreter = interpreter
elif copied_interpreter != interpreter:
raise RuntimeError('Detected binaries with different interpreters: %s != %s' %(copied_interpreter, interpreter))
@@ -659,7 +565,7 @@
log_level = logging.getLevelName(LOG_MESSAGE)
handler = LogHandler(sys.stdout)
- logger = logging.getLogger(__name__)
+ logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(log_level)
return handler
Modified: trunk/Tools/Scripts/run-jsc-benchmarks (279983 => 279984)
--- trunk/Tools/Scripts/run-jsc-benchmarks 2021-07-16 11:16:13 UTC (rev 279983)
+++ trunk/Tools/Scripts/run-jsc-benchmarks 2021-07-16 14:25:26 UTC (rev 279984)
@@ -237,6 +237,7 @@
$includeTailBench = true
$includeBigIntBench = false
$includePrivateFieldsBench = false
+$ldd=nil
$measureGC=false
$benchmarkPattern=nil
$verbosity=0
@@ -1413,11 +1414,28 @@
@extraEnv[key] = val
@@extraEnvSet[key] = true
end
-
+
+ def copyIntoBenchPathBundle(basename, outputdir)
+ bundle_binary = (SCRIPT_PATH.dirname + 'bundle-binary').realpath
+ cmd = Shellwords.join([
+ bundle_binary,
+ '--dest-dir', outputdir,
+ @path
+ ])
+ $stderr.puts ">> #{cmd}" if $verbosity>=2
+ raise unless system(cmd)
+ @path = "#{basename}/jsc"
+ end
+
def copyIntoBenchPath
raise unless canCopyIntoBenchPath
basename, filename = Benchfile.uniqueFilename("vm")
raise unless Dir.mkdir(filename)
+ if not $remoteHosts.empty? and `uname` == 'Linux'
+ copyIntoBenchPathBundle(basename, Shellwords.shellescape(filename.to_s))
+ return
+ end
+
@libPath.each {
| libPathPart |
cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
@@ -1424,7 +1442,10 @@
$stderr.puts ">> #{cmd}" if $verbosity>=2
raise unless system(cmd)
}
- @path = "#{basename}/#{@relativeBinPath}"
+ cmd = "cp -a #{@path} #{Shellwords.shellescape(filename.to_s)}"
+ $stderr.puts ">> #{cmd}" if $verbosity>=2
+ raise unless system(cmd)
+ @path = "#{basename}/jsc"
@libPath = [basename]
end
@@ -2389,7 +2410,17 @@
}
result + curLine + "\n"
end
-
+
+def exportJSCEnvironmentVariables
+ buf = ""
+ ENV.each {
+ | name, value |
+ next unless name.start_with?("JSC_")
+ buf << "export #{name}=#{Shellwords.shellescape(value)}\n"
+ }
+ buf
+end
+
def runAndGetResults
results = nil
Dir.chdir(BENCH_DATA_PATH) {
@@ -3055,6 +3086,8 @@
$dependencies.push(Pathname.new(arg).realpath)
when '--config'
$configPath = Pathname.new(arg)
+ when '--ldd'
+ $ldd = arg
when '--help'
usage
else
@@ -3611,6 +3644,7 @@
if $prepare
File.open("#{BENCH_DATA_PATH}/runscript", "w") {
| file |
+ file.puts exportJSCEnvironmentVariables
file.puts "echo -e \"HOSTNAME:\\c\""
file.puts "hostname"
file.puts "echo"
Modified: trunk/Tools/Scripts/run-jsc-stress-tests (279983 => 279984)
--- trunk/Tools/Scripts/run-jsc-stress-tests 2021-07-16 11:16:13 UTC (rev 279983)
+++ trunk/Tools/Scripts/run-jsc-stress-tests 2021-07-16 14:25:26 UTC (rev 279984)
@@ -1919,7 +1919,7 @@
end
if $remote and $hostOS == "linux"
- generate_bundle = (Pathname.new(THIS_SCRIPT_PATH).dirname + 'generate-bundle').realpath
+ bundle_binary = (Pathname.new(THIS_SCRIPT_PATH).dirname + 'bundle-binary').realpath
Dir.mktmpdir {
| tmpdir |
# Generate bundle in a temporary directory so that
@@ -1926,30 +1926,15 @@
# we can safely pick it up regardless of its name
# (it's the only zip file there).
cmdline = [
- generate_bundle.to_s,
- "--platform=gtk",
- "--bundle=jsc",
- "--syslibs=bundle-all",
- "--no-strip",
- "--compression=tar.xz",
- ($buildType == "release") ? "--release" : "--debug",
- "--destination=#{tmpdir}"
+ bundle_binary.to_s,
+ "--dest-dir=#{$jscPath.dirname}",
+ "--log-level=debug",
+ $jscPath.to_s
]
if not $ldd.nil?
cmdline << "--ldd=#{$ldd}"
end
mysys(cmdline)
- archives = Dir.glob("#{tmpdir}/*.tar.xz")
- if archives.size != 1
- raise "Expected exactly one entry in tmpdir, not #{archives}"
- end
- # Note: we overwrite 'jsc'. This obviously conflicts with
- # !copyVM but, then gain, so does $remote.
- mysys(["tar",
- "-C",
- $jscPath.dirname.to_s,
- "-xf",
- archives[0]])
}
end
}
Added: trunk/Tools/Scripts/webkitpy/binary_bundling/__init__.py (0 => 279984)
--- trunk/Tools/Scripts/webkitpy/binary_bundling/__init__.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/binary_bundling/__init__.py 2021-07-16 14:25:26 UTC (rev 279984)
@@ -0,0 +1,13 @@
+# Required for Python to search this directory for module files
+
+# Keep this file free of any code or import statements that could
+# cause either an error to occur or a log message to be logged.
+# This ensures that calling code can import initialization code from
+# webkitpy before any errors or log messages due to code in this file.
+# Initialization code can include things like version-checking code and
+# logging configuration code.
+#
+# We do not execute any version-checking code or logging configuration
+# code in this file so that callers can opt-in as they want. This also
+# allows different callers to choose different initialization code,
+# as necessary.
Added: trunk/Tools/Scripts/webkitpy/binary_bundling/bundle.py (0 => 279984)
--- trunk/Tools/Scripts/webkitpy/binary_bundling/bundle.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/binary_bundling/bundle.py 2021-07-16 14:25:26 UTC (rev 279984)
@@ -0,0 +1,98 @@
+# Copyright (C) 2018, 2020, 2021 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import logging
+import os
+import shutil
+import subprocess
+
+_log = logging.getLogger(__name__)
+
+
+class BinaryBundler:
+ VAR_MYDIR = 'MYDIR'
+ VAR_INTERPRETER = 'INTERPRETER'
+
+ def __init__(self, destination_dir, should_strip_objects):
+ if shutil.which('patchelf') is None:
+ _log.error("Could not find `patchelf` in $PATH")
+ raise Exception("Missing required binary `patchelf`")
+ self._destination_dir = destination_dir
+ self._should_strip_objects = should_strip_objects
+
+ def destination_dir(self):
+ return self._destination_dir
+
+ def copy_and_remove_rpath(self, orig_file, type='bin', destination_dir=None):
+ dir_suffix = 'lib' if type == 'interpreter' else type
+ destination_dir = os.path.join(self._destination_dir, dir_suffix)
+ if not os.path.isdir(destination_dir):
+ os.makedirs(destination_dir)
+
+ if not os.path.isfile(orig_file):
+ raise ValueError('Can not find file %s' % orig_file)
+
+ _log.info('Add to bundle [%s]: %s' % (type, orig_file))
+ try:
+ shutil.copy(orig_file, destination_dir)
+ except shutil.SameFileError:
+ # May reasonably happen if the caller tries to bundle the files 'in place'.
+ pass
+
+ patch_elf_command = ['patchelf', '--remove-rpath', os.path.join(destination_dir, os.path.basename(orig_file))]
+ if subprocess.call(patch_elf_command) != 0:
+ _log.error('The patchelf command returned non-zero status')
+
+ if self._should_strip_objects:
+ if shutil.which('strip'):
+ strip_command = ['strip', '--strip-unneeded', os.path.join(destination_dir, os.path.basename(orig_file))]
+ if subprocess.call(strip_command) != 0:
+ _log.error('The strip command returned non-zero status')
+ else:
+ _log.warning('strip not found. Not stripping object')
+
+ def generate_wrapper_script(self, interpreter, binary_to_wrap, extra_environment_variables={}):
+ if not os.path.isfile(os.path.join(self._destination_dir, 'bin', binary_to_wrap)):
+ raise RuntimeError('Cannot find binary to wrap for %s' % binary_to_wrap)
+ _log.info('Generate wrapper script %s' % binary_to_wrap)
+ script_file = os.path.join(self._destination_dir, binary_to_wrap)
+
+ with open(script_file, 'w') as script_handle:
+ script_handle.write('#!/bin/sh\n')
+ script_handle.write('%s="$(dirname $(readlink -f $0))"\n' % self.VAR_MYDIR)
+ script_handle.write('export LD_LIBRARY_PATH="${%s}/lib"\n' % self.VAR_MYDIR)
+
+ if interpreter is not None:
+ script_handle.write('INTERPRETER="${%s}/lib/%s"\n' % (self.VAR_MYDIR, os.path.basename(interpreter)))
+
+ # May use the value of INTERPRETER, so has to come after
+ # the definition.
+ for var, value in extra_environment_variables.items():
+ script_handle.write('export %s=\"%s\"\n' % (var, value))
+
+ if interpreter is not None:
+ script_handle.write('exec "${INTERPRETER}" "${%s}/bin/%s" "$@"\n' % (self.VAR_MYDIR, binary_to_wrap))
+ else:
+ script_handle.write('exec "${%s}/bin/%s" "$@"\n' % (self.VAR_MYDIR, binary_to_wrap))
+
+ os.chmod(script_file, 0o755)
Added: trunk/Tools/Scripts/webkitpy/binary_bundling/ldd.py (0 => 279984)
--- trunk/Tools/Scripts/webkitpy/binary_bundling/ldd.py (rev 0)
+++ trunk/Tools/Scripts/webkitpy/binary_bundling/ldd.py 2021-07-16 14:25:26 UTC (rev 279984)
@@ -0,0 +1,109 @@
+# Copyright (C) 2018, 2020, 2021 Igalia S.L.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+# list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+from pathlib import PurePath
+import os
+import shutil
+import subprocess
+
+_log = logging.getLogger(__name__)
+
+
+class SharedObjectResolver():
+ def __init__(self, ldd):
+ self._ldd = ldd
+ if shutil.which('patchelf') is None:
+ _log.error("Could not find `patchelf` in $PATH")
+ raise Exception("Missing required binary `patchelf`")
+
+ def _run_cmd_and_get_output(self, command):
+ _log.debug("EXEC %s" % command)
+ command_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8')
+ stdout, stderr = command_process.communicate()
+ return command_process.returncode, stdout, stderr
+
+ def _get_interpreter_objname(self, object):
+ # Note: we use patchelf to get the object name (not the path!)
+ # of the interpreter because this works regardless of the
+ # architecture of the ELF file.
+ retcode, stdout, stderr = self._run_cmd_and_get_output(['patchelf', '--print-interpreter', object])
+ if retcode != 0:
+ _log.debug("patchelf stdout:\n%s\nPatchelf stderr:\n%s" % (stdout, stderr))
+ if 'cannot find section' in stdout:
+ # This is fine; we only expect an interpreter in the main binary.
+ return None
+ raise RuntimeError('The patchelf command returned non-zero status for object %s' % object)
+ interpreter_path = PurePath(stdout.strip())
+ return interpreter_path.name
+
+ def _get_libs_and_interpreter(self, object):
+ interpreter = None
+ retcode, stdout, stderr = self._run_cmd_and_get_output([self._ldd, object])
+ _log.debug("ldd stdout:\n%s" % stdout)
+ if retcode != 0:
+ raise RuntimeError('The %s command returned non-zero status for object %s' % (self._ldd, object))
+ libs = []
+ for line in stdout.splitlines():
+ line = line.strip()
+ if '=>' in line:
+ line = line.split('=>')[1].strip()
+ if 'not found' in line:
+ raise RuntimeError('ldd can not resolve all dependencies for object %s.' % object)
+ line = line.split(' ')[0].strip()
+ if os.path.isfile(line):
+ libs.append(line)
+ else:
+ line = line.split(' ')[0].strip()
+ if os.path.isfile(line):
+ interpreter = line
+ if interpreter is None:
+ # This is the case for non-native binaries. For those, we
+ # can use a cross-ldd (xldd), but then the interpreter
+ # looks like any other shared object in the output of
+ # ldd. Try to identify it by looking at the object name
+ # from the interpreter string.
+ interpreter_objname = self._get_interpreter_objname(object)
+ for lib in libs:
+ if PurePath(lib).name == interpreter_objname:
+ interpreter = lib
+ break
+ # If we found an interpreter, remove it from the libs.
+ libs = [lib for lib in libs if lib != interpreter]
+ return libs, interpreter
+
+ def _ldd_recursive_get_libs_and_interpreter(self, object, already_checked_libs=[]):
+ libs, interpreter = self._get_libs_and_interpreter(object)
+ if libs:
+ for lib in libs:
+ if lib in already_checked_libs:
+ continue
+ # avoid recursion loops (libfreetype.so.6 <-> libharfbuzz.so.0)
+ already_checked_libs.append(lib)
+ sub_libs, sub_interpreter = self._ldd_recursive_get_libs_and_interpreter(lib, already_checked_libs)
+ libs.extend(sub_libs)
+ if sub_interpreter and interpreter and sub_interpreter != interpreter:
+ raise RuntimeError('library %s has interpreter %s but object %s has interpreter %s' % (lib, sub_interpreter, object, interpreter))
+ return list(set(libs)), interpreter
+
+ def get_libs_and_interpreter(self, object, already_checked_libs=[]):
+ return self._ldd_recursive_get_libs_and_interpreter(object, already_checked_libs)
