Commit: 6f516fcc631c177a1c9f62b3a22e0c4e0e9713d4 Author: Brecht Van Lommel Date: Fri May 10 23:00:35 2019 +0200 Branches: master https://developer.blender.org/rB6f516fcc631c177a1c9f62b3a22e0c4e0e9713d4
Tests: speed up render tests by running multiple in the same process Blender startup time and shader compilation is a big factor when running hundreds of tests, so now all renders in the same ctest run in the same process. This was previously reverted due to skipping other tests when one test crashed. Now if a test crashes, Blender is re-run with the remaining tests so we get results from them still. =================================================================== M tests/python/cycles_render_tests.py M tests/python/eevee_render_tests.py M tests/python/modules/render_report.py M tests/python/opengl_draw_tests.py M tests/python/workbench_render_tests.py =================================================================== diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 349ba83f88a..79ba11fdd44 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -9,14 +9,13 @@ import subprocess import sys -def render_file(filepath, output_filepath): +def get_arguments(filepath, output_filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) - frame_filepath = output_filepath + '0001.png' - - common_args = [ + args = [ + "--background", "-noaudio", "--factory-startup", "--enable-autoexec", @@ -29,57 +28,17 @@ def render_file(filepath, output_filepath): # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] custom_args = os.getenv('CYCLESTEST_ARGS') - custom_args = shlex.split(custom_args) if custom_args else [] - common_args += custom_args + if custom_args: + args.extend(shlex.split(custom_args)) - if subject == 'opengl': - command = [BLENDER, "--window-geometry", "0", "0", "1", "1"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_opengl.py")] - elif subject == 'bake': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_bake.py")] + if subject == 'bake': + args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")]) elif subject == 'denoise_animation': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_denoise.py")] + args.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")]) else: - command = [BLENDER, "--background"] - command += common_args - command += ["-f", "1"] - - try: - # Success - output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e) - return "CRASH" + args.extend(["-f", "1"]) + return args def create_argparse(): parser = argparse.ArgumentParser() @@ -94,11 +53,7 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] @@ -108,7 +63,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("cycles_renders") report.set_compare_engines('cycles', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments, batch=True) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index 9fb8a5f41a8..0875e4b171a 100755 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys - def setup(): import bpy @@ -46,15 +45,8 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - command = [ - BLENDER, +def get_arguments(filepath, output_filepath): + return [ "--background", "-noaudio", "--factory-startup", @@ -67,37 +59,6 @@ def render_file(filepath, output_filepath): "-F", "PNG", "-f", "1"] - try: - # Success - output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e) - return "CRASH" - def create_argparse(): parser = argparse.ArgumentParser() @@ -112,11 +73,7 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] @@ -126,7 +83,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("eevee_renders") report.set_compare_engines('eevee', 'cycles') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments, batch=True) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 2f99a3b6292..15826f97400 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -134,10 +134,10 @@ class Report: def set_compare_engines(self, engine, other_engine): self.compare_engines = (engine, other_engine) - def run(self, dirpath, render_cb): + def run(self, dirpath, blender, arguments_cb, batch=False): # Run tests and output report. dirname = os.path.basename(dirpath) - ok = self._run_all_tests(dirname, dirpath, render_cb) + ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch) self._write_data(dirname) self._write_html() if self.compare_engines: @@ -399,43 +399,81 @@ class Report: return not failed - def _run_test(self, filepath, render_cb): - testname = test_get_name(filepath) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname) + def _run_tests(self, filepaths, blender, arguments_cb, batch): + # Run multiple tests in a single Blender process since startup can be + # a significant factor. In case of crashes, re-run the remaining tests. + verbose = os.environ.get("BLENDER_VERBOSE") is not None - error = render_cb(filepath, tmp_filepath) - status = "FAIL" - if not error: - if not self._diff_output(filepath, tmp_filepath): - error = "VERIFY" + remaining_filepaths = filepaths[:] + errors = [] - if os.path.exists(tmp_filepath): - os.remove(tmp_filepath) + while len(remaining_filepaths) > 0: + command = [blender] + output_filepaths = [] - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_ENGINE": - print_message("Can't perform tests because the render engine failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error + # Construct output filepaths and command to run + for filepath in remaining_filepaths: + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') - def _run_all_tests(self, dirname, dirpath, render_cb): + base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname) + output_filepath = base_output_filepath + '0001.png' + output_filepaths.append(output_filepath) + + if os.path.exists(output_filepath): + os.remove(output_filepath) + + command.extend(arguments_cb(filepath, base_output_filepath)) + + # Only chain multiple commands for batch + if not batch: + break + + # Run process + crash = False + try: + output = subprocess.check_out @@ Diff output truncated at 10240 characters. @@ _______________________________________________ Bf-blender-cvs mailing list Bf-blender-cvs@blender.org https://lists.blender.org/mailman/listinfo/bf-blender-cvs